From 10bccf6b5687da8db4e8e5bc2b8fe1edb0ddaeab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 15 Jan 2021 16:26:51 +0100 Subject: [PATCH] Better component handling in the project manager (#1399) --- .github/workflows/release.yml | 194 +++++------ .github/workflows/scala.yml | 14 +- build.sbt | 18 +- distribution/enso.bundle.template | 1 + docs/distribution/README.md | 4 +- docs/distribution/bundles.md | 71 ++++ ...allback-launcher-release-infrastructure.md | 2 +- docs/distribution/local-repository.md | 107 ------ docs/distribution/standard-libraries.md | 2 +- .../org/enso/projectmanager/boot/Cli.scala | 34 +- .../projectmanager/boot/ProjectManager.scala | 41 +-- ...imeVersionManagerErrorRecoverySyntax.scala | 17 +- .../DefaultDistributionConfiguration.scala | 48 +-- ...untimeVersionManagementUserInterface.scala | 2 +- .../components/LocalReleaseProviderSpec.scala | 168 ---------- .../RuntimeVersionManagerSpec.scala | 142 +++++++- .../DistributionManagerSpec.scala | 24 +- .../components/ComponentsException.scala | 4 + .../components/Engine.scala | 6 +- .../components/GraalRuntime.scala | 2 +- .../components/RuntimeVersionManager.scala | 104 ++++-- .../distribution/DistributionManager.scala | 59 +++- .../PortableDistributionManager.scala | 6 + .../releases/engine/EngineRepository.scala | 14 - .../graalvm/GraalCEReleaseProvider.scala | 12 - .../releases/local/LocalReleaseProvider.scala | 112 ------- project/DistributionPackage.scala | 316 +++++++++++++++++- tools/ci/prepare-distribution-env.sh | 13 +- 28 files changed, 830 insertions(+), 707 deletions(-) create mode 100644 distribution/enso.bundle.template create mode 100644 docs/distribution/bundles.md delete mode 100644 docs/distribution/local-repository.md delete mode 100644 lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/LocalReleaseProviderSpec.scala delete mode 100644 lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/local/LocalReleaseProvider.scala diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 63d1d4eebb..1e12786d33 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,7 +31,7 @@ jobs: with: path: repo - name: Enable Developer Command Prompt (Windows) - uses: ilammy/msvc-dev-cmd@v1.4.1 + uses: ilammy/msvc-dev-cmd@v1.5.0 - name: Disable TCP/UDP Offloading (macOS) if: runner.os == 'macOS' shell: bash @@ -147,19 +147,19 @@ jobs: shell: bash run: | sleep 1 - sbt 'set launcherDistributionRoot := file("${{ env.LAUNCHER_DIST_DIR }}"); buildLauncherDistribution' + sbt buildLauncherDistribution - name: Prepare Engine Distribution working-directory: repo shell: bash run: | sleep 1 - sbt 'set engineDistributionRoot := file("${{ env.ENGINE_DIST_DIR }}"); buildEngineDistribution' + sbt buildEngineDistribution - name: Prepare Project Manager Distribution working-directory: repo shell: bash run: | sleep 1 - sbt 'set projectManagerDistributionRoot := file("${{ env.PROJECTMANAGER_DIST_DIR }}"); buildProjectManagerDistribution' + sbt buildProjectManagerDistribution # Ensure that the versions encoded in the binary and in the release match - name: Check Versions (Unix) @@ -202,17 +202,17 @@ jobs: - name: Upload the Engine Artifact uses: actions/upload-artifact@v2 with: - name: ${{ env.ENGINE_DIST_ROOT }} + name: ${{ env.ENGINE_DIST_NAME }} path: repo/${{ env.ENGINE_DIST_ROOT }} - name: Upload the Launcher Artifact uses: actions/upload-artifact@v2 with: - name: ${{ env.LAUNCHER_DIST_ROOT }} + name: ${{ env.LAUNCHER_DIST_NAME }} path: repo/${{ env.LAUNCHER_DIST_ROOT }} - name: Upload the Project Manager Artifact uses: actions/upload-artifact@v2 with: - name: ${{ env.PROJECTMANAGER_DIST_ROOT }} + name: ${{ env.PROJECTMANAGER_DIST_NAME }} path: repo/${{ env.PROJECTMANAGER_DIST_ROOT }} - name: Upload the Manifest Artifact uses: actions/upload-artifact@v2 @@ -237,12 +237,36 @@ jobs: # Without specifying options, it downloads all artifacts - uses: actions/download-artifact@v2 with: - path: artifacts + path: repo/built-distribution # This jobs can be used to debug errors, it may be removed - name: Display Structure of Downloaded Files run: ls -R - working-directory: artifacts + working-directory: repo/built-distribution + + - name: Setup GraalVM Environment + uses: ayltai/setup-graalvm@v1 + with: + graalvm-version: ${{ env.graalVersion }} + java-version: ${{ env.javaVersion }} + native-image: true + - name: Set Up SBT + shell: bash + run: | + curl -fsSL -o sbt.tgz https://github.com/sbt/sbt/releases/download/v${{env.sbtVersion}}/sbt-${{env.sbtVersion}}.tgz + tar -xzf sbt.tgz + echo $GITHUB_WORKSPACE/sbt/bin/ >> $GITHUB_PATH + + # Caches + - name: Cache SBT + uses: actions/cache@v2 + with: + path: | + ~/.sbt + ~/.ivy2/cache + ~/.cache + key: ${{ runner.os }}-sbt-${{ hashFiles('**build.sbt') }} + restore-keys: ${{ runner.os }}-sbt- - name: Save Version to Environment shell: bash @@ -252,87 +276,19 @@ jobs: echo "Preparing release for $DIST_VERSION" echo "DIST_VERSION=$DIST_VERSION" >> $GITHUB_ENV - - name: Download GraalVM for Bundles - shell: bash - run: | - curl -fsSL -o graalvm-linux.tar.gz "https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-${{ env.graalVersion }}/graalvm-ce-java${{ env.javaVersion }}-linux-amd64-${{ env.graalVersion }}.tar.gz" - echo "Linux JVM downloaded" - curl -fsSL -o graalvm-macos.tar.gz "https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-${{ env.graalVersion }}/graalvm-ce-java${{ env.javaVersion }}-darwin-amd64-${{ env.graalVersion }}.tar.gz" - echo "MacOS JVM downloaded" - curl -fsSL -o graalvm-windows.zip "https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-${{ env.graalVersion }}/graalvm-ce-java${{ env.javaVersion }}-windows-amd64-${{ env.graalVersion }}.zip" - echo "Windows JVM downloaded" - mkdir graalvm-linux - mkdir graalvm-macos - mkdir graalvm-windows - (cd graalvm-linux && tar xf ../graalvm-linux.tar.gz) - echo "Linux JVM extracted" - (cd graalvm-macos && tar xf ../graalvm-macos.tar.gz) - echo "MacOS JVM extracted" - (cd graalvm-windows && unzip -q ../graalvm-windows.zip) - echo "Windows JVM extracted" - - # As the download-artifact action does not preserve the executable bits, - # we fix them here, so that the release assets are easy to use. - - name: Fix Package Structure - shell: bash - run: | - chmod +x artifacts/enso-engine-${{ env.DIST_VERSION }}-linux-amd64/enso-${{ env.DIST_VERSION }}/bin/enso - chmod +x artifacts/enso-engine-${{ env.DIST_VERSION }}-macos-amd64/enso-${{ env.DIST_VERSION }}/bin/enso - chmod +x artifacts/enso-launcher-${{ env.DIST_VERSION }}-linux-amd64/enso/bin/enso - chmod +x artifacts/enso-launcher-${{ env.DIST_VERSION }}-macos-amd64/enso/bin/enso - chmod +x artifacts/enso-project-manager-${{ env.DIST_VERSION }}-linux-amd64/enso/bin/project-manager - chmod +x artifacts/enso-project-manager-${{ env.DIST_VERSION }}-macos-amd64/enso/bin/project-manager - mkdir artifacts/enso-launcher-${{ env.DIST_VERSION }}-linux-amd64/enso/config - mkdir artifacts/enso-launcher-${{ env.DIST_VERSION }}-linux-amd64/enso/dist - mkdir artifacts/enso-launcher-${{ env.DIST_VERSION }}-linux-amd64/enso/runtime - mkdir artifacts/enso-launcher-${{ env.DIST_VERSION }}-macos-amd64/enso/config - mkdir artifacts/enso-launcher-${{ env.DIST_VERSION }}-macos-amd64/enso/dist - mkdir artifacts/enso-launcher-${{ env.DIST_VERSION }}-macos-amd64/enso/runtime - mkdir artifacts/enso-launcher-${{ env.DIST_VERSION }}-windows-amd64/enso/config - mkdir artifacts/enso-launcher-${{ env.DIST_VERSION }}-windows-amd64/enso/dist - mkdir artifacts/enso-launcher-${{ env.DIST_VERSION }}-windows-amd64/enso/runtime - - name: Prepare Packages shell: bash + working-directory: repo run: | - (cd artifacts/enso-engine-${{ env.DIST_VERSION }}-linux-amd64/ && tar -czf ../../enso-engine-${{ env.DIST_VERSION }}-linux-amd64.tar.gz enso-${{ env.DIST_VERSION }} ) - echo "Linux Engine packaged" - (cd artifacts/enso-engine-${{ env.DIST_VERSION }}-macos-amd64/ && tar -czf ../../enso-engine-${{ env.DIST_VERSION }}-macos-amd64.tar.gz enso-${{ env.DIST_VERSION }} ) - echo "MacOS Engine packaged" - (cd artifacts/enso-engine-${{ env.DIST_VERSION }}-windows-amd64/ && zip -q -r ../../enso-engine-${{ env.DIST_VERSION }}-windows-amd64.zip enso-${{ env.DIST_VERSION }} ) - echo "Windows Engine packaged" - - (cd artifacts/enso-launcher-${{ env.DIST_VERSION }}-linux-amd64/ && tar -czf ../../enso-launcher-${{ env.DIST_VERSION }}-linux-amd64.tar.gz enso ) - echo "Linux Launcher packaged" - (cd artifacts/enso-launcher-${{ env.DIST_VERSION }}-macos-amd64/ && tar -czf ../../enso-launcher-${{ env.DIST_VERSION }}-macos-amd64.tar.gz enso ) - echo "MacOS Launcher packaged" - (cd artifacts/enso-launcher-${{ env.DIST_VERSION }}-windows-amd64/ && zip -q -r ../../enso-launcher-${{ env.DIST_VERSION }}-windows-amd64.zip enso ) - echo "Windows Launcher packaged" - - (cd artifacts/enso-project-manager-${{ env.DIST_VERSION }}-linux-amd64/ && tar -czf ../../enso-project-manager-${{ env.DIST_VERSION }}-linux-amd64.tar.gz enso ) - echo "Linux Project Manager packaged" - (cd artifacts/enso-project-manager-${{ env.DIST_VERSION }}-macos-amd64/ && tar -czf ../../enso-project-manager-${{ env.DIST_VERSION }}-macos-amd64.tar.gz enso ) - echo "MacOS Project Manager packaged" - (cd artifacts/enso-project-manager-${{ env.DIST_VERSION }}-windows-amd64/ && zip -q -r ../../enso-project-manager-${{ env.DIST_VERSION }}-windows-amd64.zip enso ) - echo "Windows Project Manager packaged" + sleep 1 + sbt makePackages - name: Prepare Bundles shell: bash + working-directory: repo run: | - cp -r artifacts/enso-engine-${{ env.DIST_VERSION }}-linux-amd64/enso-${{ env.DIST_VERSION }} artifacts/enso-launcher-${{ env.DIST_VERSION }}-linux-amd64/enso/dist/${{ env.DIST_VERSION }} - cp -r artifacts/enso-engine-${{ env.DIST_VERSION }}-macos-amd64/enso-${{ env.DIST_VERSION }} artifacts/enso-launcher-${{ env.DIST_VERSION }}-macos-amd64/enso/dist/${{ env.DIST_VERSION }} - cp -r artifacts/enso-engine-${{ env.DIST_VERSION }}-windows-amd64/enso-${{ env.DIST_VERSION }} artifacts/enso-launcher-${{ env.DIST_VERSION }}-windows-amd64/enso/dist/${{ env.DIST_VERSION }} - mv graalvm-linux/graalvm-ce-java${{ env.javaVersion }}-${{ env.graalVersion }} artifacts/enso-launcher-${{ env.DIST_VERSION }}-linux-amd64/enso/runtime - mv graalvm-macos/graalvm-ce-java${{ env.javaVersion }}-${{ env.graalVersion }} artifacts/enso-launcher-${{ env.DIST_VERSION }}-macos-amd64/enso/runtime - mv graalvm-windows/graalvm-ce-java${{ env.javaVersion }}-${{ env.graalVersion }} artifacts/enso-launcher-${{ env.DIST_VERSION }}-windows-amd64/enso/runtime - - echo "Bundles prepared" - (cd artifacts/enso-launcher-${{ env.DIST_VERSION }}-linux-amd64/ && tar -czf ../../enso-bundle-${{ env.DIST_VERSION }}-linux-amd64.tar.gz enso ) - echo "Linux Bundle packaged" - (cd artifacts/enso-launcher-${{ env.DIST_VERSION }}-macos-amd64/ && tar -czf ../../enso-bundle-${{ env.DIST_VERSION }}-macos-amd64.tar.gz enso ) - echo "MacOS Bundle packaged" - (cd artifacts/enso-launcher-${{ env.DIST_VERSION }}-windows-amd64/ && zip -q -r ../../enso-bundle-${{ env.DIST_VERSION }}-windows-amd64.zip enso ) - echo "Windows Bundle packaged" + sleep 1 + sbt makeBundles - name: Create Release id: create_release @@ -357,7 +313,7 @@ jobs: shell: bash run: | sleep 1 - docker build -t ensosharedwus2acr.azurecr.io/runtime:${{ env.DIST_VERSION }} -f ./repo/tools/ci/docker/Dockerfile ./artifacts/enso-engine-${{ env.DIST_VERSION }}-linux-amd64/enso-${{ env.DIST_VERSION }}/component + docker build -t ensosharedwus2acr.azurecr.io/runtime:${{ env.DIST_VERSION }} -f ./repo/tools/ci/docker/Dockerfile ./repo/built-distribution/enso-engine-${{ env.DIST_VERSION }}-linux-amd64/enso-${{ env.DIST_VERSION }}/component docker push ensosharedwus2acr.azurecr.io/runtime:${{ env.DIST_VERSION }} # Publish the launcher packages to the backup/fallback S3 bucket @@ -373,25 +329,26 @@ jobs: - name: Upload the Linux Launcher Package to S3 shell: bash run: > - aws s3 cp enso-launcher-${{ env.DIST_VERSION }}-linux-amd64.tar.gz - s3://launcherfallback/launcher/enso-${{ env.DIST_VERSION }}/ --profile - s3-upload --acl public-read + aws s3 cp repo/built-distribution/enso-launcher-${{ env.DIST_VERSION + }}-linux-amd64.tar.gz s3://launcherfallback/launcher/enso-${{ + env.DIST_VERSION }}/ --profile s3-upload --acl public-read - name: Upload the macOS Launcher Package to S3 shell: bash run: > - aws s3 cp enso-launcher-${{ env.DIST_VERSION }}-macos-amd64.tar.gz - s3://launcherfallback/launcher/enso-${{ env.DIST_VERSION }}/ --profile - s3-upload --acl public-read + aws s3 cp repo/built-distribution/enso-launcher-${{ env.DIST_VERSION + }}-macos-amd64.tar.gz s3://launcherfallback/launcher/enso-${{ + env.DIST_VERSION }}/ --profile s3-upload --acl public-read - name: Upload the Windows Launcher Package to S3 shell: bash run: > - aws s3 cp enso-launcher-${{ env.DIST_VERSION }}-windows-amd64.zip - s3://launcherfallback/launcher/enso-${{ env.DIST_VERSION }}/ --profile - s3-upload --acl public-read + aws s3 cp repo/built-distribution/enso-launcher-${{ env.DIST_VERSION + }}-windows-amd64.zip s3://launcherfallback/launcher/enso-${{ + env.DIST_VERSION }}/ --profile s3-upload --acl public-read - name: Upload the Launcher Manifest to S3 shell: bash run: > - aws s3 cp artifacts/launcher-manifest/launcher-manifest.yaml + aws s3 cp + repo/built-distribution/launcher-manifest/launcher-manifest.yaml s3://launcherfallback/launcher/enso-${{ env.DIST_VERSION }}/launcher-manifest.yaml --profile s3-upload --acl public-read - name: Update the Release List in S3 @@ -422,7 +379,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: enso-engine-${{ env.DIST_VERSION }}-linux-amd64.tar.gz + asset_path: + repo/built-distribution/enso-engine-${{ env.DIST_VERSION + }}-linux-amd64.tar.gz asset_name: enso-engine-${{ env.DIST_VERSION }}-linux-amd64.tar.gz asset_content_type: application/x-tar - name: Publish the Engine (MacOS) @@ -431,7 +390,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: enso-engine-${{ env.DIST_VERSION }}-macos-amd64.tar.gz + asset_path: + repo/built-distribution/enso-engine-${{ env.DIST_VERSION + }}-macos-amd64.tar.gz asset_name: enso-engine-${{ env.DIST_VERSION }}-macos-amd64.tar.gz asset_content_type: application/x-tar - name: Publish the Engine (Windows) @@ -440,7 +401,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: enso-engine-${{ env.DIST_VERSION }}-windows-amd64.zip + asset_path: + repo/built-distribution/enso-engine-${{ env.DIST_VERSION + }}-windows-amd64.zip asset_name: enso-engine-${{ env.DIST_VERSION }}-windows-amd64.zip asset_content_type: application/zip @@ -450,7 +413,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: enso-launcher-${{ env.DIST_VERSION }}-linux-amd64.tar.gz + asset_path: + repo/built-distribution/enso-launcher-${{ env.DIST_VERSION + }}-linux-amd64.tar.gz asset_name: enso-launcher-${{ env.DIST_VERSION }}-linux-amd64.tar.gz asset_content_type: application/x-tar - name: Publish the Launcher (MacOS) @@ -459,7 +424,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: enso-launcher-${{ env.DIST_VERSION }}-macos-amd64.tar.gz + asset_path: + repo/built-distribution/enso-launcher-${{ env.DIST_VERSION + }}-macos-amd64.tar.gz asset_name: enso-launcher-${{ env.DIST_VERSION }}-macos-amd64.tar.gz asset_content_type: application/x-tar - name: Publish the Launcher (Windows) @@ -468,7 +435,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: enso-launcher-${{ env.DIST_VERSION }}-windows-amd64.zip + asset_path: + repo/built-distribution/enso-launcher-${{ env.DIST_VERSION + }}-windows-amd64.zip asset_name: enso-launcher-${{ env.DIST_VERSION }}-windows-amd64.zip asset_content_type: application/zip @@ -479,7 +448,8 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: - enso-project-manager-${{ env.DIST_VERSION }}-linux-amd64.tar.gz + repo/built-distribution/enso-project-manager-${{ env.DIST_VERSION + }}-linux-amd64.tar.gz asset_name: enso-project-manager-${{ env.DIST_VERSION }}-linux-amd64.tar.gz asset_content_type: application/x-tar @@ -490,7 +460,8 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: - enso-project-manager-${{ env.DIST_VERSION }}-macos-amd64.tar.gz + repo/built-distribution/enso-project-manager-${{ env.DIST_VERSION + }}-macos-amd64.tar.gz asset_name: enso-project-manager-${{ env.DIST_VERSION }}-macos-amd64.tar.gz asset_content_type: application/x-tar @@ -501,7 +472,8 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: - enso-project-manager-${{ env.DIST_VERSION }}-windows-amd64.zip + repo/built-distribution/enso-project-manager-${{ env.DIST_VERSION + }}-windows-amd64.zip asset_name: enso-project-manager-${{ env.DIST_VERSION }}-windows-amd64.zip asset_content_type: application/zip @@ -512,7 +484,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: enso-bundle-${{ env.DIST_VERSION }}-linux-amd64.tar.gz + asset_path: + repo/built-distribution/enso-bundle-${{ env.DIST_VERSION + }}-linux-amd64.tar.gz asset_name: enso-bundle-${{ env.DIST_VERSION }}-linux-amd64.tar.gz asset_content_type: application/x-tar - name: Publish the Bundle (MacOS) @@ -521,7 +495,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: enso-bundle-${{ env.DIST_VERSION }}-macos-amd64.tar.gz + asset_path: + repo/built-distribution/enso-bundle-${{ env.DIST_VERSION + }}-macos-amd64.tar.gz asset_name: enso-bundle-${{ env.DIST_VERSION }}-macos-amd64.tar.gz asset_content_type: application/x-tar - name: Publish the Bundle (Windows) @@ -530,7 +506,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: enso-bundle-${{ env.DIST_VERSION }}-windows-amd64.zip + asset_path: + repo/built-distribution/enso-bundle-${{ env.DIST_VERSION + }}-windows-amd64.zip asset_name: enso-bundle-${{ env.DIST_VERSION }}-windows-amd64.zip asset_content_type: application/zip @@ -540,7 +518,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: artifacts/manifest/manifest.yaml + asset_path: repo/built-distribution/manifest/manifest.yaml asset_name: manifest.yaml asset_content_type: application/yaml - name: Publish the Launcher Manifest @@ -549,6 +527,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: artifacts/launcher-manifest/launcher-manifest.yaml + asset_path: repo/built-distribution/launcher-manifest/launcher-manifest.yaml asset_name: launcher-manifest.yaml asset_content_type: application/yaml diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index c59846074e..06e750ffcf 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -39,7 +39,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Enable Developer Command Prompt (Windows) - uses: ilammy/msvc-dev-cmd@v1.4.1 + uses: ilammy/msvc-dev-cmd@v1.5.0 - name: Setup Go uses: actions/setup-go@v2 - name: Disable TCP/UDP Offloading (macOS) @@ -188,7 +188,7 @@ jobs: shell: bash run: | sleep 1 - sbt 'set launcherDistributionRoot := file("${{ env.LAUNCHER_DIST_DIR }}"); buildLauncherDistribution' + sbt buildLauncherDistribution # The way artifacts are uploaded currently does not preserve the # executable bits for Unix. However putting artifacts into a ZIP would @@ -201,13 +201,13 @@ jobs: shell: bash run: | sleep 1 - sbt 'set engineDistributionRoot := file("${{ env.ENGINE_DIST_DIR }}"); buildEngineDistribution' + sbt buildEngineDistribution - name: Prepare Project Manager Distribution shell: bash run: | sleep 1 - sbt 'set projectManagerDistributionRoot := file("${{ env.PROJECTMANAGER_DIST_DIR }}"); buildProjectManagerDistribution' + sbt buildProjectManagerDistribution # Test Distribution - name: Prepare Engine Test Environment @@ -243,17 +243,17 @@ jobs: - name: Publish the Engine Distribution Artifact uses: actions/upload-artifact@v2 with: - name: ${{ env.ENGINE_DIST_ROOT }} + name: ${{ env.ENGINE_DIST_NAME }} path: ${{ env.ENGINE_DIST_ROOT }} - name: Publish the Launcher uses: actions/upload-artifact@v2 with: - name: ${{ env.LAUNCHER_DIST_ROOT }} + name: ${{ env.LAUNCHER_DIST_NAME }} path: ${{ env.LAUNCHER_DIST_ROOT }} - name: Publish the Project Manager uses: actions/upload-artifact@v2 with: - name: ${{ env.PROJECTMANAGER_DIST_ROOT }} + name: ${{ env.PROJECTMANAGER_DIST_NAME }} path: ${{ env.PROJECTMANAGER_DIST_ROOT }} - name: Prepare the FlatBuffers Schemas for Upload diff --git a/build.sbt b/build.sbt index c57334d373..57fc5033f6 100644 --- a/build.sbt +++ b/build.sbt @@ -93,6 +93,13 @@ openLegalReviewReport := { lazy val analyzeDependency = inputKey[Unit]("...") analyzeDependency := GatherLicenses.analyzeDependency.evaluated +val packageBuilder = new DistributionPackage.Builder( + ensoVersion = ensoVersion, + graalVersion = graalVersion, + graalJavaVersion = javaVersion, + artifactRoot = file("built-distribution") +) + Global / onChangedBuildSource := ReloadOnSourceChanges // ============================================================================ @@ -201,6 +208,9 @@ lazy val enso = (project in file(".")) testkit ) .settings(Global / concurrentRestrictions += Tags.exclusive(Exclusive)) + .settings( + commands ++= Seq(packageBuilder.makePackages, packageBuilder.makeBundles) + ) // ============================================================================ // === Dependency Versions ==================================================== @@ -1338,9 +1348,11 @@ lazy val launcherDistributionRoot = lazy val projectManagerDistributionRoot = settingKey[File]("Root of built project manager distribution") -engineDistributionRoot := file("built-distribution/engine") -launcherDistributionRoot := file("built-distribution/launcher") -projectManagerDistributionRoot := file("built-distribution/project-manager") +engineDistributionRoot := + packageBuilder.localArtifact("engine") / s"enso-$ensoVersion" +launcherDistributionRoot := packageBuilder.localArtifact("launcher") / "enso" +projectManagerDistributionRoot := + packageBuilder.localArtifact("project-manager") / "enso" lazy val buildEngineDistribution = taskKey[Unit]("Builds the engine distribution") diff --git a/distribution/enso.bundle.template b/distribution/enso.bundle.template new file mode 100644 index 0000000000..fcf66b49c2 --- /dev/null +++ b/distribution/enso.bundle.template @@ -0,0 +1 @@ +Enso Bundle Marker diff --git a/docs/distribution/README.md b/docs/distribution/README.md index 0d8491c2ad..d2d5841fa3 100644 --- a/docs/distribution/README.md +++ b/docs/distribution/README.md @@ -27,7 +27,7 @@ dependencies, and Enso projects for use by our users. Explanation of the fallback infrastructure that can be enabled to keep launcher updates functioning even if the primary release provider stops working. -- [**Local Repository:**](local-repository.md) Explanation of local repository - structure that is used for bundling engine with project manager distributions. - [**Standard Libraries:**](standard-libraries.md) A brief explanation of the standard libraries for Enso. +- [**Bundles**](bundles.md) An explanation of distributed bundles that contain + all components necessary to run Enso out of the box. diff --git a/docs/distribution/bundles.md b/docs/distribution/bundles.md new file mode 100644 index 0000000000..4e1362d0c9 --- /dev/null +++ b/docs/distribution/bundles.md @@ -0,0 +1,71 @@ +--- +layout: developer-doc +title: Distribution Bundles +category: distribution +tags: [distribution, layout, bundles] +order: 9 +--- + +# Bundles + +This document describes how the distributions are bundled to provide releases +that work out-of-the box, allowing to use the latest engine without downloading +any additional dependencies. + + + +- [Project Manager Bundle](#project-manager-bundle) +- [Launcher Bundles](#launcher-bundles) + + + +## Project Manager Bundle + +The Project Manager is distributed with latest engine version and its +corresponding Graal runtime to avoid having to download them at first startup. + +The bundled components are placed in their respective subdirectories (not as +packages, but extracted and ready to use) and a bundle marker file called +`.enso.bundle` must be placed next to these directories so that the Project +Manager can detect the bundle. + +The `project-manager` executable looks for the `.enso.bundle` marker in the +parent directory of the directory that it is, itself, located in. So overall, +the bundle should have the following structure (the actual engine and Graal +versions may of course differ): + +``` +enso +├── bin +│ └── project-manager +├── dist +│ └── 0.2.1-SNAPSHOT +├── other-project-manager-files +└── runtime + └── graalvm-ce-java11-20.2.0 +``` + +If the bundle is detected, the additional `dist` and `runtime` directories are +added as secondary search paths for components. Thus, the `project-manager` can +use both components present in the default +[installed location](distribution.md#installed-enso-distribution-layout) or +those from the bundle. In a situation that the same component were to be +available both in the installed location and the bundle, the installed location +is preferred. New components are installed in the installed location, never next +to the bundles. + +In fact, it is possible for the bundle directory to be read-only (which may be +the case for example if the Project Manager bundle is packaged as part of IDE's +AppImage package). In such situation, it will be impossible to uninstall the +bundled components and a relevant error message will be returned. + +## Launcher Bundles + +Bundles are also distributed for the launcher, but these are implemented using a +different mechanism. + +Since the launcher can run in +[portable mode](distribution.md#portable-enso-distribution-layout), the bundled +engine and runtime are simply included within its portable package. They can +then be used from within this portable package or +[installed](distribution.md#installing-from-a-portable-distribution). diff --git a/docs/distribution/fallback-launcher-release-infrastructure.md b/docs/distribution/fallback-launcher-release-infrastructure.md index 54d8b13191..0b341bb5c1 100644 --- a/docs/distribution/fallback-launcher-release-infrastructure.md +++ b/docs/distribution/fallback-launcher-release-infrastructure.md @@ -3,7 +3,7 @@ layout: developer-doc title: Fallback Launcher Release Infrastructure category: distribution tags: [distribution, launcher, fallback] -order: 6 +order: 7 --- # Fallback Launcher Release Infrastructure diff --git a/docs/distribution/local-repository.md b/docs/distribution/local-repository.md deleted file mode 100644 index 5aca74a246..0000000000 --- a/docs/distribution/local-repository.md +++ /dev/null @@ -1,107 +0,0 @@ ---- -layout: developer-doc -title: Local Repository -category: distribution -tags: [distribution, project-manager, offline, local] -order: 7 ---- - -# Local Repository - -A `LocalReleaseProvider` is implemented that allows to install components from -local (offline) repositories. This functionality can be used to allow installing -bundled components. - - - -- [Local Repository Structure](#local-repository-structure) - - [General Repository Structure](#general-repository-structure) - - [Engine Repository Structure](#engine-repository-structure) - - [GraalVM Repository Structure](#graalvm-repository-structure) -- [Usage in Project Manager](#usage-in-project-manager) - - - -## Local Repository Structure - -This section explains how a local repository has to be structured in order to -work with `LocalReleaseProvider`. - -### General Repository Structure - -In general a local repository should be a separate directory that contains only -directories corresponding to releases of a single component. Repositories for -separate components should be kept separately. For each provided release it -should contain a directory called after the release's tag. That directory of -each release should just contain assets associated with that release. - -### Engine Repository Structure - -The engine repository contains a directory `enso-` for each release -that resembles the GitHub release structure. The directory for each release -should contain a `manifest.yaml` file and the package. As separate bundles are -created for each operating system, only the package for the desired operating -system is required (normally releases contain packages for all supported -systems). It's naming scheme is the same as in GitHub releases, that is -`enso-engine---.{zip|tar.gz}`. The extension is `zip` for -Windows and `tar.gz` for other platforms. Currently, the only supported `ARCH` -is `amd64`. The `OS` can be one of `windows`, `linux`, `macos`. - -For example, a local engine repository could look like this: - -``` -localengine -└── enso-0.1.2-rc.9 - ├── enso-engine-0.1.2-rc.9-linux-amd64.tar.gz - └── manifest.yaml -``` - -### GraalVM Repository Structure - -The GraalVM repository contains a directory `vm-` for each release. -Inside of that directory, a package should be included (again as in case of the -engine, only the package for the current operating system is required). The -package name should be -`graalvm-ce-java---.{zip|tar.gz}`. The -extension is `zip` on Windows and `tar.gz` on other platforms. The `ARCH` should -be the same as for the engine. The `OS` can be one of `windows`, `linux`, -`darwin`. **Note that the MacOS package has different naming scheme for GraalVM -than it has for the engine**. - -For example, a local GraalVM repository can look like this: - -``` -localruntime -└── vm-20.2.0 - └── graalvm-ce-java11-linux-amd64-20.2.0.tar.gz -``` - -## Usage in Project Manager - -Command line options can be used to enable the local repositories in the project -manager. `--local-engine-repository` sets the path to the engine repository and -`--local-graal-repository` sets the path to the GraalVM repository. The provided -paths should be absolute to be sure that they are resolved correctly. - -For example, starting the project manager as written below will first look for -engines and GraalVM runtimes in the provided local repositories (but if they are -not found, online repository will be used as fallback, if it is available). - -```bash -./project-manager --local-engine-repository /a/b/bundle/engines --local-graal-repository /a/b/bundle/graalvm -``` - -For the above command to work properly, the directory `/a/b/bundle/` may have -the following structure: - -``` -bundle -├── other files (project-manager binary etc.) -├── engines -│ └── enso-0.1.2-rc.9 -│ ├── enso-engine-0.1.2-rc.9-linux-amd64.tar.gz -│ └── manifest.yaml -└── graalvm - └── vm-20.2.0 - └── graalvm-ce-java11-linux-amd64-20.2.0.tar.gz -``` diff --git a/docs/distribution/standard-libraries.md b/docs/distribution/standard-libraries.md index 7df1771188..e769270750 100644 --- a/docs/distribution/standard-libraries.md +++ b/docs/distribution/standard-libraries.md @@ -3,7 +3,7 @@ layout: developer-doc title: Standard Libraries category: distribution tags: [distribution, stdlib] -order: 9 +order: 8 --- # Standard Libraries diff --git a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/boot/Cli.scala b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/boot/Cli.scala index b6947d4f88..5a29c3a0f9 100644 --- a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/boot/Cli.scala +++ b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/boot/Cli.scala @@ -7,12 +7,10 @@ import scala.util.Try object Cli { - val JSON_OPTION = "json" - val HELP_OPTION = "help" - val VERBOSE_OPTION = "verbose" - val VERSION_OPTION = "version" - val LOCAL_ENGINE_REPOSITORY = "local-engine-repository" - val LOCAL_GRAAL_REPOSITORY = "local-graal-repository" + val JSON_OPTION = "json" + val HELP_OPTION = "help" + val VERBOSE_OPTION = "verbose" + val VERSION_OPTION = "version" object option { @@ -37,28 +35,6 @@ object Cli { .longOpt(JSON_OPTION) .desc("Switches the --version option to JSON output.") .build() - - val localEngineRepository: cli.Option = cli.Option.builder - .longOpt(LOCAL_ENGINE_REPOSITORY) - .hasArg - .numberOfArgs(1) - .argName("path") - .desc( - "Allows the Project Manager to install engine versions from an " + - "offline repository." - ) - .build() - - val localGraalRepository: cli.Option = cli.Option.builder - .longOpt(LOCAL_GRAAL_REPOSITORY) - .hasArg - .numberOfArgs(1) - .argName("path") - .desc( - "Allows the Project Manager to install GraalVM versions from an " + - "offline repository." - ) - .build() } val options: cli.Options = @@ -67,8 +43,6 @@ object Cli { .addOption(option.verbose) .addOption(option.version) .addOption(option.json) - .addOption(option.localEngineRepository) - .addOption(option.localGraalRepository) /** Parse the command line options. */ def parse(args: Array[String]): Either[String, cli.CommandLine] = { diff --git a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/boot/ProjectManager.scala b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/boot/ProjectManager.scala index f074c2d601..02bca4720b 100644 --- a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/boot/ProjectManager.scala +++ b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/boot/ProjectManager.scala @@ -1,7 +1,6 @@ package org.enso.projectmanager.boot import java.io.IOException -import java.nio.file.{InvalidPathException, Path} import java.util.concurrent.ScheduledThreadPoolExecutor import akka.http.scaladsl.Http @@ -15,7 +14,6 @@ import org.enso.projectmanager.boot.Globals.{ SuccessExitCode } import org.enso.projectmanager.boot.configuration.ProjectManagerConfig -import org.enso.projectmanager.versionmanagement.DefaultDistributionConfiguration import org.enso.version.VersionDescription import pureconfig.ConfigSource import pureconfig.generic.auto._ @@ -120,44 +118,7 @@ object ProjectManager extends App with LazyLogging { } else { val verbosity = options.getOptions.count(_ == Cli.option.verbose) logger.info("Starting Project Manager...") - - def parsePath(string: String) = ZIO.effect(Path.of(string)) - def parseOptionalPath(string: Option[String]) = - string.map(parsePath).map(_.map(Some(_))).getOrElse(ZIO.succeed(None)) - - val initializeLocalRepositories = for { - enginePath <- parseOptionalPath( - Option(options.getOptionValue(Cli.LOCAL_ENGINE_REPOSITORY)) - ) - graalPath <- parseOptionalPath( - Option(options.getOptionValue(Cli.LOCAL_GRAAL_REPOSITORY)) - ) - _ <- ZIO.effect( - DefaultDistributionConfiguration.setupLocalRepositories( - enginePath, - graalPath - ) - ) - } yield () - - val initializeRepositoryOrLogError = initializeLocalRepositories - .catchSome { case error: InvalidPathException => - ZIO.effectTotal( - logger - .error(s"Could not parse a local repository path: $error", error) - ) - } - .catchAll { th => - ZIO.effectTotal( - logger.error( - "Failed to initialize local repositories, " + - "default ones will be used.", - th - ) - ) - } - - setupLogging(verbosity) *> initializeRepositoryOrLogError *> + setupLogging(verbosity) *> mainProcess.fold( th => { logger.error("Main process execution failed.", th) diff --git a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/service/versionmanagement/RuntimeVersionManagerErrorRecoverySyntax.scala b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/service/versionmanagement/RuntimeVersionManagerErrorRecoverySyntax.scala index 54902f4645..25fcad56d3 100644 --- a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/service/versionmanagement/RuntimeVersionManagerErrorRecoverySyntax.scala +++ b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/service/versionmanagement/RuntimeVersionManagerErrorRecoverySyntax.scala @@ -2,19 +2,8 @@ package org.enso.projectmanager.service.versionmanagement import org.enso.projectmanager.control.effect.ErrorChannel import org.enso.projectmanager.service.ProjectServiceFailure -import org.enso.projectmanager.service.ProjectServiceFailure.{ - BrokenComponentFailure, - ComponentInstallationFailure, - MissingComponentFailure, - ProjectManagerUpgradeRequiredFailure -} -import org.enso.runtimeversionmanager.components.{ - BrokenComponentError, - ComponentMissingError, - ComponentsException, - InstallationError, - UpgradeRequiredError -} +import org.enso.projectmanager.service.ProjectServiceFailure._ +import org.enso.runtimeversionmanager.components._ object RuntimeVersionManagerErrorRecoverySyntax { implicit class ErrorRecovery[F[+_, +_]: ErrorChannel, A]( @@ -42,6 +31,8 @@ object RuntimeVersionManagerErrorRecoverySyntax { ProjectManagerUpgradeRequiredFailure( upgradeRequired.expectedVersion ) + case UninstallationError(message) => + ComponentUninstallationFailure(message) case _ => mapDefault(componentsException) } case other: Throwable => diff --git a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala index 2ace0d4189..0a3083144d 100644 --- a/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala +++ b/lib/scala/project-manager/src/main/scala/org/enso/projectmanager/versionmanagement/DefaultDistributionConfiguration.scala @@ -1,7 +1,5 @@ package org.enso.projectmanager.versionmanagement -import java.nio.file.Path - import com.typesafe.scalalogging.LazyLogging import org.enso.runtimeversionmanager.Environment import org.enso.runtimeversionmanager.components.{ @@ -19,10 +17,7 @@ import org.enso.runtimeversionmanager.releases.engine.{ EngineRelease, EngineRepository } -import org.enso.runtimeversionmanager.releases.graalvm.{ - GraalCEReleaseProvider, - GraalVMRuntimeReleaseProvider -} +import org.enso.runtimeversionmanager.releases.graalvm.GraalCEReleaseProvider import org.enso.runtimeversionmanager.runner.JVMSettings /** Default distribution configuration to use for the Project Manager in @@ -52,15 +47,9 @@ object DefaultDistributionConfiguration lazy val temporaryDirectoryManager = new TemporaryDirectoryManager(distributionManager, resourceManager) - private var currentEngineReleaseProvider: ReleaseProvider[EngineRelease] = - EngineRepository.defaultEngineReleaseProvider - /** @inheritdoc */ def engineReleaseProvider: ReleaseProvider[EngineRelease] = - currentEngineReleaseProvider - - private var runtimeReleaseProvider: GraalVMRuntimeReleaseProvider = - GraalCEReleaseProvider.default + EngineRepository.defaultEngineReleaseProvider /** @inheritdoc */ def makeRuntimeVersionManager( @@ -72,7 +61,7 @@ object DefaultDistributionConfiguration temporaryDirectoryManager = temporaryDirectoryManager, resourceManager = resourceManager, engineReleaseProvider = engineReleaseProvider, - runtimeReleaseProvider = runtimeReleaseProvider, + runtimeReleaseProvider = GraalCEReleaseProvider.default, installerKind = InstallerKind.ProjectManager ) @@ -81,35 +70,4 @@ object DefaultDistributionConfiguration /** @inheritdoc */ override def shouldDiscardChildOutput: Boolean = false - - /** Sets up local repositories if they were requested. - * @param engineRepositoryPath the path to a local engine repository if one - * should be used - * @param graalRepositoryPath the path to a local GraalVM repository if one - * should be used - */ - def setupLocalRepositories( - engineRepositoryPath: Option[Path], - graalRepositoryPath: Option[Path] - ): Unit = { - val engineProviderOverride = - engineRepositoryPath.map(path => - (path, EngineRepository.fromLocalRepository(path)) - ) - - val graalProviderOverride = - graalRepositoryPath.map(path => - (path, GraalCEReleaseProvider.fromLocalRepository(path)) - ) - - engineProviderOverride.foreach { case (path, newProvider) => - logger.debug(s"Using a local engine repository from $path.") - currentEngineReleaseProvider = newProvider - } - - graalProviderOverride.foreach { case (path, newProvider) => - logger.debug(s"Using a local GraalVM repository from $path.") - runtimeReleaseProvider = newProvider - } - } } diff --git a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/TestRuntimeVersionManagementUserInterface.scala b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/TestRuntimeVersionManagementUserInterface.scala index 89825ee441..c1b9b12a62 100644 --- a/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/TestRuntimeVersionManagementUserInterface.scala +++ b/lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/TestRuntimeVersionManagementUserInterface.scala @@ -38,7 +38,7 @@ class TestRuntimeVersionManagementUserInterface(installBroken: Boolean) true /** @inheritdoc */ - override def logInfo(message: => String): Unit = logger.debug(message) + override def logInfo(message: => String): Unit = logger.info(message) override def startWaitingForResource(resource: Resource): Unit = logger.debug(s"Waiting on ${resource.name}") diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/LocalReleaseProviderSpec.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/LocalReleaseProviderSpec.scala deleted file mode 100644 index 738b213669..0000000000 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/LocalReleaseProviderSpec.scala +++ /dev/null @@ -1,168 +0,0 @@ -package org.enso.runtimeversionmanager.components - -import java.nio.file.Files - -import nl.gn0s1s.bump.SemVer -import org.enso.loggingservice.TestLogger -import org.enso.runtimeversionmanager.FileSystem -import org.enso.runtimeversionmanager.FileSystem.PathSyntax -import org.enso.runtimeversionmanager.releases.{ - EnsoReleaseProvider, - Release, - SimpleReleaseProvider -} -import org.enso.runtimeversionmanager.releases.engine.EngineReleaseProvider -import org.enso.runtimeversionmanager.releases.graalvm.GraalCEReleaseProvider -import org.enso.runtimeversionmanager.releases.local.LocalReleaseProvider -import org.enso.runtimeversionmanager.releases.testing.TestArchivePackager -import org.enso.runtimeversionmanager.test.{ - FakeReleases, - RuntimeVersionManagerTest -} - -import scala.util.{Failure, Try} - -class LocalReleaseProviderSpec extends RuntimeVersionManagerTest { - private def localEngines = getTestDirectory / "offline-engine" - private def localRuntimes = getTestDirectory / "offline-graal" - - /** Creates a [[RuntimeVersionManager]] that is tied to the local/offline - * repository with fallback to default fake release repository. - */ - private def makeRuntimeManagerWithLocalRepository(): RuntimeVersionManager = { - val engineProvider = - new LocalReleaseProvider(localEngines, FakeReleases.baseEngineProvider) - val runtimeProvider = - new LocalReleaseProvider(localRuntimes, FakeReleases.baseRuntimeProvider) - makeManagers( - engineProvider = new EngineReleaseProvider(engineProvider), - runtimeProvider = new GraalCEReleaseProvider(runtimeProvider) - )._2 - } - - /** Creates a local repository with engine 1.2.3-local that uses GraalVM 20.20.20-local. - * - * It uses the prepared fake-releases and just re-configures them. - */ - private def prepareLocalRepository(): Unit = { - Files.createDirectories(localEngines) - Files.createDirectories(localRuntimes) - - { - val engineVersion = SemVer(1, 2, 3, Some("local")) - val engineName = s"enso-$engineVersion" - val destinationRoot = localEngines / engineName - Files.createDirectories(destinationRoot) - val manifest = - """minimum-launcher-version: 0.0.1 - |minimum-project-manager-version: 0.0.1 - |graal-vm-version: 20.20.20-local - |graal-java-version: 11 - |""".stripMargin - val sourceArchive = - EnsoReleaseProvider.packageNameForComponent("engine", SemVer(0, 0, 0)) - val tmpArchive = getTestDirectory / sourceArchive - FileSystem.copyDirectory( - FakeReleases.engineRoot / "enso-0.0.0" / sourceArchive / "enso-0.0.0", - tmpArchive / engineName - ) - FileSystem.writeTextFile( - tmpArchive / engineName / "manifest.yaml", - manifest - ) - FileSystem.writeTextFile(destinationRoot / "manifest.yaml", manifest) - val archive = - EnsoReleaseProvider.packageNameForComponent("engine", engineVersion) - TestArchivePackager.packArchive( - tmpArchive, - destinationRoot / archive - ) - } - - { - val sourceVersion = - GraalVMVersion(SemVer(2, 0, 0), "11") - val targetVersion = - GraalVMVersion(SemVer(20, 20, 20, Some("local")), "11") - val destinationRoot = localRuntimes / s"vm-${targetVersion.graalVersion}" - Files.createDirectories(destinationRoot) - val sourceRoot = - FakeReleases.runtimeRoot / s"vm-${sourceVersion.graalVersion}" - TestArchivePackager.packArchive( - sourceRoot / GraalCEReleaseProvider.packageFileNameForCurrentOS( - sourceVersion - ), - destinationRoot / GraalCEReleaseProvider.packageFileNameForCurrentOS( - targetVersion - ) - ) - } - } - - "LocalReleaseProvider" should { - "install a release from a local repository" in { - prepareLocalRepository() - val runtimeVersionManager = makeRuntimeManagerWithLocalRepository() - val engineVersion = SemVer(1, 2, 3, Some("local")) - val runtimeVersion = - GraalVMVersion(SemVer(20, 20, 20, Some("local")), "11") - runtimeVersionManager.findOrInstallEngine(engineVersion) - runtimeVersionManager - .listInstalledEngines() - .map(_.version) shouldEqual Seq(engineVersion) - runtimeVersionManager - .listInstalledGraalRuntimes() - .map(_.version) shouldEqual Seq(runtimeVersion) - } - - "install a release from the fallback repository" in { - val runtimeVersionManager = makeRuntimeManagerWithLocalRepository() - val engineVersion = SemVer(0, 0, 0) - runtimeVersionManager.findOrInstallEngine(engineVersion) - runtimeVersionManager - .listInstalledEngines() - .map(_.version) shouldEqual Seq(engineVersion) - } - - "include releases from both local and fallback" in { - val localVersion = "enso-1.2.3-local" - Files.createDirectories(localEngines / localVersion) - - val releaseProvider = new LocalReleaseProvider( - localEngines, - FakeReleases.baseEngineProvider - ) - - val tags = releaseProvider.listReleases().get.map(_.tag) - tags should contain(localVersion) - tags should contain("enso-0.0.0") - } - - "work in 'offline-mode' if fallback is unavailable" in { - val unavailableProvider = new SimpleReleaseProvider { - override def releaseForTag(tag: String): Try[Release] = - Failure(new RuntimeException("Repository unavailable.")) - - override def listReleases(): Try[Seq[Release]] = - Failure(new RuntimeException("Repository unavailable.")) - } - - val localVersion = "enso-1.2.3-local" - Files.createDirectories(localEngines / localVersion) - val releaseProvider = - new LocalReleaseProvider(localEngines, unavailableProvider) - - val logs = TestLogger.gatherLogs { - releaseProvider.listReleases().get.map(_.tag) shouldEqual Seq( - localVersion - ) - - Thread.sleep(500) // making sure the log is processed - } - val expectedMessage = - "The remote release provider failed with java.lang.RuntimeException: " + - "Repository unavailable., but locally bundled releases are available." - logs.map(_.message) should contain(expectedMessage) - } - } -} diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala index 8e2cbebc79..ba42c4eb37 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManagerSpec.scala @@ -1,14 +1,18 @@ package org.enso.runtimeversionmanager.components +import java.nio.file.{Files, Path} + import nl.gn0s1s.bump.SemVer -import org.enso.runtimeversionmanager.components +import org.enso.runtimeversionmanager.FileSystem.PathSyntax import org.enso.runtimeversionmanager.config.GlobalConfigurationManager import org.enso.runtimeversionmanager.test.{ RuntimeVersionManagerTest, TestRuntimeVersionManagementUserInterface } +import org.enso.runtimeversionmanager.{components, FileSystem, OS} +import org.enso.testkit.OsSpec -class RuntimeVersionManagerSpec extends RuntimeVersionManagerTest { +class RuntimeVersionManagerSpec extends RuntimeVersionManagerTest with OsSpec { "RuntimeVersionManager" should { "find the latest engine version in semver ordering " + @@ -176,5 +180,139 @@ class RuntimeVersionManagerSpec extends RuntimeVersionManagerTest { } upgradeException.expectedVersion shouldEqual bigVersion } + + "support bundled components" in { + val engineVersion = SemVer(0, 1, 0) + val runtimeVersion = GraalVMVersion(SemVer(1, 0, 0), "11") + prepareBundle( + engines = Seq(engineVersion), + runtimes = Seq(runtimeVersion) + ) + val manager = makeRuntimeVersionManager() + + val engine = manager.findEngine(engineVersion).value + engine.version shouldEqual engineVersion + engine.isMarkedBroken shouldEqual false + engine.ensureValid() + + manager.findGraalRuntime(engine).value.version shouldEqual runtimeVersion + manager.findGraalRuntime(runtimeVersion).value.ensureValid() + } + + "fail to uninstall a read-only bundled component" taggedAs OsUnix in { + val engineVersion = SemVer(0, 1, 0) + val runtimeVersion = GraalVMVersion(SemVer(1, 0, 0), "11") + prepareBundle( + engines = Seq(engineVersion), + runtimes = Seq(runtimeVersion) + ) + val manager = makeRuntimeVersionManager() + def installedEngines = manager.listInstalledEngines().map(_.version) + def installedRuntimes = + manager.listInstalledGraalRuntimes().map(_.version) + + val enginePath = getTestDirectory / "dist" / "0.1.0" + val runtimePath = getTestDirectory / "runtime" / "graalvm-ce-java11-1.0.0" + + enginePath.toFile.setWritable(false) + try { + intercept[UninstallationError] { + manager.uninstallEngine(engineVersion) + } + + installedEngines shouldEqual Seq(engineVersion) + installedRuntimes shouldEqual Seq(runtimeVersion) + } finally { + enginePath.toFile.setWritable(true) + } + + runtimePath.toFile.setWritable(false) + try { + manager.uninstallEngine(engineVersion) + + installedEngines shouldEqual Seq() + installedRuntimes shouldEqual Seq(runtimeVersion) + + manager.cleanupRuntimes() + installedRuntimes shouldEqual Seq(runtimeVersion) + } finally { + runtimePath.toFile.setWritable(true) + } + + manager.cleanupRuntimes() + installedRuntimes shouldEqual Seq() + } + + "include both bundled and installed components in list" in { + prepareBundle( + engines = Seq(SemVer(0, 0, 1)), + runtimes = Seq(GraalVMVersion(SemVer(1, 0, 0), "11")) + ) + val manager = makeRuntimeVersionManager() + manager.findOrInstallEngine(SemVer(0, 1, 0)) + + manager + .listInstalledEngines() + .map(_.version) should contain theSameElementsAs Seq( + SemVer(0, 0, 1), + SemVer(0, 1, 0) + ) + + val runtimeVersions = manager.listInstalledGraalRuntimes().map(_.version) + runtimeVersions.map(_.graalVersion) should contain theSameElementsAs Seq( + SemVer(1, 0, 0), + SemVer(2, 0, 0) + ) + runtimeVersions.map(_.java).toSet shouldEqual Set("11") + } + } + + private def prepareBundle( + engines: Seq[SemVer], + runtimes: Seq[GraalVMVersion] + ): Unit = { + FileSystem.writeTextFile( + getTestDirectory / ".enso.bundle", + "Enso Bundle Marker" + ) + for (engineVersion <- engines) { + fakeInstallEngine(getTestDirectory / "dist", engineVersion) + } + for (runtimeVersion <- runtimes) { + fakeInstallRuntime(getTestDirectory / "runtime", runtimeVersion) + } + } + + private def fakeInstallEngine(searchPath: Path, version: SemVer): Unit = { + val manifest = """minimum-launcher-version: 0.0.1 + |minimum-project-manager-version: 0.0.1 + |graal-vm-version: 1.0.0 + |graal-java-version: 11""".stripMargin + val root = searchPath / version.toString + Files.createDirectories(root) + FileSystem.writeTextFile(root / "manifest.yaml", manifest) + val components = root / "component" + Files.createDirectories(components) + makePlaceholder(components / "runner.jar") + FileSystem.writeTextFile(components / "runtime.jar", "placeholder") + } + + private def fakeInstallRuntime( + searchPath: Path, + version: GraalVMVersion + ): Unit = { + val root = + searchPath / s"graalvm-ce-java${version.java}-${version.graalVersion}" + val bin = + if (OS.operatingSystem == OS.MacOS) root / "Contents" / "Home" / "bin" + else root / "bin" + Files.createDirectories(bin) + val executable = if (OS.isWindows) "java.exe" else "java" + makePlaceholder(bin / executable) + } + + private def makePlaceholder(path: Path): Unit = { + FileSystem.writeTextFile(path, "placeholder") + path.toFile.setExecutable(true) } } diff --git a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/distributuion/DistributionManagerSpec.scala b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/distributuion/DistributionManagerSpec.scala index 609182740f..0084baa21a 100644 --- a/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/distributuion/DistributionManagerSpec.scala +++ b/lib/scala/runtime-version-manager-test/src/test/scala/org/enso/runtimeversionmanager/distributuion/DistributionManagerSpec.scala @@ -1,8 +1,8 @@ package org.enso.runtimeversionmanager.distributuion -import java.nio.file.Path +import java.nio.file.{Files, Path} -import org.enso.runtimeversionmanager.Environment +import org.enso.runtimeversionmanager.{Environment, FileSystem} import org.enso.runtimeversionmanager.FileSystem.PathSyntax import org.enso.runtimeversionmanager.distribution.{ DistributionManager, @@ -12,6 +12,7 @@ import org.enso.runtimeversionmanager.test.{ FakeEnvironment, WithTemporaryDirectory } +import org.scalatest.OptionValues import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -19,7 +20,8 @@ class DistributionManagerSpec extends AnyWordSpec with Matchers with WithTemporaryDirectory - with FakeEnvironment { + with FakeEnvironment + with OptionValues { "DistributionManager" should { "detect portable distribution" in { @@ -35,6 +37,7 @@ class DistributionManagerSpec distributionManager.paths.runtimes shouldEqual getTestDirectory / "runtime" distributionManager.paths.engines shouldEqual getTestDirectory / "dist" + distributionManager.paths.bundle shouldEqual None } "detect installed distribution" in { @@ -45,6 +48,21 @@ class DistributionManagerSpec val distributionManager = new PortableDistributionManager(fakeEnvironment) distributionManager.isRunningPortable shouldEqual false + distributionManager.paths.bundle shouldEqual None + } + + "detect bundles" in { + val executable = fakeExecutablePath() + FileSystem.writeTextFile(getTestDirectory / ".enso.bundle", "placeholder") + + val fakeEnvironment = new Environment { + override def getPathToRunningExecutable: Path = executable + } + + val distributionManager = new PortableDistributionManager(fakeEnvironment) + val bundle = distributionManager.paths.bundle.value + assert(Files.isSameFile(bundle.engines, getTestDirectory / "dist")) + assert(Files.isSameFile(bundle.runtimes, getTestDirectory / "runtime")) } "respect environment variable overrides " + diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/ComponentsException.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/ComponentsException.scala index 453506d59d..d7367218ad 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/ComponentsException.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/ComponentsException.scala @@ -65,3 +65,7 @@ case class UpgradeRequiredError( s"Minimum version required to use this engine is " + s"$expectedVersion." ) + +/** Indicates uninstallation failure. */ +case class UninstallationError(message: String) + extends ComponentsException(message) diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Engine.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Engine.scala index 7efdf1d348..d777487201 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Engine.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/Engine.scala @@ -48,14 +48,14 @@ case class Engine(version: SemVer, path: Path, manifest: Manifest) { Failure( CorruptedComponentError( s"Engine's runner.jar (expected at " + - s"${runnerPath.toAbsolutePath.normalize} is missing." + s"`${runnerPath.toAbsolutePath.normalize}`) is missing." ) ) else if (!Files.exists(runtimePath)) Failure( CorruptedComponentError( - s"Engine's runtime.jar (expected at " + - s"${runtimePath.toAbsolutePath.normalize} is missing." + s"`Engine's runtime.jar (expected at " + + s"${runtimePath.toAbsolutePath.normalize}`) is missing." ) ) else Success(()) diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalRuntime.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalRuntime.scala index 3906eb2670..d85aa2c93e 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalRuntime.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/GraalRuntime.scala @@ -43,7 +43,7 @@ case class GraalRuntime(version: GraalVMVersion, path: Path) { Failure( CorruptedComponentError( s"Runtime's java executable (expected at " + - s"${javaExecutable.toAbsolutePath.normalize}) is missing." + s"`${javaExecutable.toAbsolutePath.normalize}`) is missing." ) ) else if (!Files.isExecutable(javaExecutable)) diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala index db1def1fb2..28841d464f 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/components/RuntimeVersionManager.scala @@ -60,13 +60,11 @@ class RuntimeVersionManager( */ def findGraalRuntime(version: GraalVMVersion): Option[GraalRuntime] = { val name = graalRuntimeNameForVersion(version) - val path = distributionManager.paths.runtimes / name - if (Files.exists(path)) { - // TODO [RW] for now an exception is thrown if the installation is - // corrupted, in #1052 offer to repair the broken installation - loadGraalRuntime(path) - .map(Some(_)) - .recoverWith { case e: Exception => + firstExisting(distributionManager.paths.runtimeSearchPaths.map(_ / name)) + .map { path => + // TODO [RW] for now an exception is thrown if the installation is + // corrupted, in #1052 offer to repair the broken installation + loadGraalRuntime(path).recoverWith { case e: Exception => Failure( UnrecognizedComponentError( s"The runtime $version is already installed, but cannot be " + @@ -77,9 +75,8 @@ class RuntimeVersionManager( e ) ) - } - .get - } else None + }.get + } } /** Executes the provided action with a requested engine version. @@ -187,16 +184,21 @@ class RuntimeVersionManager( } } + /** Returns the first path from the sequence that exists on the file system, + * or None if no path from the sequence exists. + */ + private def firstExisting(paths: Seq[Path]): Option[Path] = + paths.find(Files.exists(_)) + /** Finds an installed engine with the given `version` and reports any errors. */ private def getEngine(version: SemVer): Try[Engine] = { val name = engineNameForVersion(version) - val path = distributionManager.paths.engines / name - if (Files.exists(path)) { - // TODO [RW] right now we return an exception, in the future (#1052) we - // will try recovery - loadEngine(path) - } else Failure(ComponentMissingError(s"Engine $version is not installed.")) + firstExisting(distributionManager.paths.engineSearchPaths.map(_ / name)) + .map(loadEngine) + .getOrElse { + Failure(ComponentMissingError(s"Engine $version is not installed.")) + } } /** Finds an engine with the given `version` or returns None if it is not @@ -224,9 +226,10 @@ class RuntimeVersionManager( Failure( UnrecognizedComponentError( s"The engine $version is already installed, but cannot be " + - s"loaded due to $e. Until the launcher gets an auto-repair " + - s"feature, please try running `enso uninstall engine $version` " + - s"followed by `enso install engine $version`.", + s"loaded due to $e " + + s"Please try reinstalling by running " + + s"`enso uninstall engine $version` followed by " + + s"`enso install engine $version`.", e ) ) @@ -270,21 +273,38 @@ class RuntimeVersionManager( /** Lists all installed GrallVM runtimes. */ def listInstalledGraalRuntimes(): Seq[GraalRuntime] = - FileSystem - .listDirectory(distributionManager.paths.runtimes) - .filter(isNotIgnoredDirectory) + findComponents(distributionManager.paths.runtimeSearchPaths) .map(path => (path, loadGraalRuntime(path))) .flatMap(handleErrorsAsWarnings[GraalRuntime]("A runtime")) /** Lists all installed engines. */ def listInstalledEngines(): Seq[Engine] = { - FileSystem - .listDirectory(distributionManager.paths.engines) - .filter(isNotIgnoredDirectory) + findComponents(distributionManager.paths.engineSearchPaths) .map(path => (path, loadEngine(path))) .flatMap(handleErrorsAsWarnings[Engine]("An engine")) } + /** Returns components found in `searchPaths`. + * + * If there are duplicate components in multiple paths, the one from the + * earliest search path is kept. + */ + private def findComponents(searchPaths: Seq[Path]): Seq[Path] = + searchPaths + .foldLeft(Map.empty[String, Path]) { case (map, searchPath) => + val componentsHere = + FileSystem.listDirectory(searchPath).filter(isNotIgnoredDirectory) + componentsHere.foldLeft(map) { case (map, componentPath) => + val componentName = componentPath.getFileName.toString + map.updatedWith(componentName) { + case Some(alreadyPresent) => Some(alreadyPresent) + case None => Some(componentPath) + } + } + } + .values + .toSeq + private def isNotIgnoredDirectory(path: Path): Boolean = { val fileName = path.getFileName.toString val isIgnored = FileSystem.ignoredFileNames.contains(fileName) @@ -328,11 +348,32 @@ class RuntimeVersionManager( throw ComponentMissingError(s"Enso Engine $version is not installed.") } + if (!Files.isWritable(engine.path)) { + val message = + s"$engine cannot be uninstalled because it is placed in a " + + s"read-only location (bundled versions cannot be uninstalled)." + logger.error(message) + throw UninstallationError(message) + } safelyRemoveComponent(engine.path) userInterface.logInfo(s"Uninstalled $engine.") - cleanupGraalRuntimes() + internalCleanupGraalRuntimes() } + /** Removes runtimes that are not used by any installed engines. + * + * Runtimes are automatically cleaned after installation, so currently this + * function is only useful for tests. + */ + def cleanupRuntimes(): Unit = { + resourceManager.withResources( + userInterface, + Resource.AddOrRemoveComponents -> LockType.Exclusive + ) { + internalCleanupGraalRuntimes() + } + } + /** Checks if the component version specified in the release's manifest is * compatible with the current installer version. */ @@ -690,14 +731,21 @@ class RuntimeVersionManager( * * The caller must hold [[Resource.AddOrRemoveComponents]] exclusively. */ - private def cleanupGraalRuntimes(): Unit = { + private def internalCleanupGraalRuntimes(): Unit = { for (runtime <- listInstalledGraalRuntimes()) { if (findEnginesUsingRuntime(runtime).isEmpty) { userInterface.logInfo( s"Removing $runtime, because it is not used by any installed Enso " + s"versions." ) - safelyRemoveComponent(runtime.path) + if (Files.isWritable(runtime.path)) { + safelyRemoveComponent(runtime.path) + } else { + logger.warn( + s"$runtime cannot be uninstalled because it is placed in a " + + s"read-only location." + ) + } } } } diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/distribution/DistributionManager.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/distribution/DistributionManager.scala index 395b3fc8a5..0aacd7ab3e 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/distribution/DistributionManager.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/distribution/DistributionManager.scala @@ -14,9 +14,12 @@ import scala.util.control.NonFatal * @param dataRoot the root of the data directory; for a portable distribution * this is the root of the distribution, for a locally * installed distribution, it corresponds to `ENSO_DATA_DIR` - * @param runtimes location of runtimes, corresponding to `runtime` directory - * @param engines location of engine versions, corresponding to `dist` + * @param runtimes primary location of runtimes, corresponding to `runtime` + * directory + * @param engines primary location of engine versions, corresponding to `dist` * directory + * @param bundle optional bundle description, containing secondary engine and + * runtime directories * @param config location of configuration * @param locks a directory for storing lockfiles that are used to synchronize * access to the various components @@ -29,6 +32,7 @@ case class DistributionPaths( dataRoot: Path, runtimes: Path, engines: Path, + bundle: Option[Bundle], config: Path, locks: Path, logs: Path, @@ -42,12 +46,36 @@ case class DistributionPaths( | dataRoot = $dataRoot, | runtimes = $runtimes, | engines = $engines, + | bundle = $bundle, | config = $config, | locks = $locks, | tmp = $unsafeTemporaryDirectory |)""".stripMargin + + /** Sequence of paths to search for engine installations, in order of + * precedence. + */ + def engineSearchPaths: Seq[Path] = Seq(engines) ++ bundle.map(_.engines).toSeq + + /** Sequence of paths to search for runtime installations, in order of + * precedence. + */ + def runtimeSearchPaths: Seq[Path] = + Seq(runtimes) ++ bundle.map(_.runtimes).toSeq } +/** Paths to secondary directories for additionally bundled engine + * distributions. + * + * These paths are only relevant for an installed distribution which may also + * use some locally bundled distributions (possibly located on a read-only + * filesystem). + * + * For portable distributions, bundled packages are already included in the + * primary directory. + */ +case class Bundle(engines: Path, runtimes: Path) + /** A helper class that encapsulates management of paths to components of the * distribution. * @@ -82,6 +110,7 @@ class DistributionManager(val env: Environment) { dataRoot = dataRoot, runtimes = dataRoot / RUNTIMES_DIRECTORY, engines = dataRoot / ENGINES_DIRECTORY, + bundle = detectBundle(), config = configRoot, locks = runRoot / LOCK_DIRECTORY, logs = LocallyInstalledDirectories.logDirectory, @@ -89,6 +118,32 @@ class DistributionManager(val env: Environment) { ) } + /** Name of the file that should be placed in the distribution root to mark it + * as running in portable mode. + */ + private val BUNDLE_MARK_FILENAME = ".enso.bundle" + + /** Root directory of a bundle. + * + * If the bundle is present, it will be located next to the `bin/` directory + * that contains the executable that we are currently running. + */ + private def possibleBundleRoot = + env.getPathToRunningExecutable.getParent.getParent + + /** Checks if [[possibleBundleRoot]] contains the bundle mark file and returns + * directories for the bundle if it was found. + */ + private def detectBundle(): Option[Bundle] = + if (Files.exists(possibleBundleRoot / BUNDLE_MARK_FILENAME)) + Some( + Bundle( + engines = possibleBundleRoot / ENGINES_DIRECTORY, + runtimes = possibleBundleRoot / RUNTIMES_DIRECTORY + ) + ) + else None + /** Removes unused lockfiles. */ def tryCleaningUnusedLockfiles(): Unit = { diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/distribution/PortableDistributionManager.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/distribution/PortableDistributionManager.scala index 5ce4e7a7cf..8678e46115 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/distribution/PortableDistributionManager.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/distribution/PortableDistributionManager.scala @@ -59,6 +59,11 @@ class PortableDistributionManager(env: Environment) portable } + /** Detects paths for the portable distribution. + * + * A portable distribution does not include bundle paths, because if + * anything was bundled with it, it is already part of its primary installation. + */ override protected def detectPaths(): DistributionPaths = if (isRunningPortable) { val root = env.getPathToRunningExecutable.getParent.getParent @@ -66,6 +71,7 @@ class PortableDistributionManager(env: Environment) dataRoot = root, runtimes = root / RUNTIMES_DIRECTORY, engines = root / ENGINES_DIRECTORY, + bundle = None, config = root / CONFIG_DIRECTORY, locks = root / LOCK_DIRECTORY, logs = root / LOG_DIRECTORY, diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineRepository.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineRepository.scala index 6294ffbbbf..1670aaa689 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineRepository.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/engine/EngineRepository.scala @@ -1,10 +1,7 @@ package org.enso.runtimeversionmanager.releases.engine -import java.nio.file.Path - import org.enso.runtimeversionmanager.releases.ReleaseProvider import org.enso.runtimeversionmanager.releases.github.GithubReleaseProvider -import org.enso.runtimeversionmanager.releases.local.LocalReleaseProvider /** Represents the default Enso repository providing releases of the engine. */ object EngineRepository { @@ -13,15 +10,4 @@ object EngineRepository { /** Default provider of engine releases. */ def defaultEngineReleaseProvider: ReleaseProvider[EngineRelease] = new EngineReleaseProvider(githubRepository) - - /** Creates an engine provider that uses a local repository first, falling - * back to the default one. - */ - def fromLocalRepository( - releaseDirectory: Path - ): ReleaseProvider[EngineRelease] = { - val mergedRepository = - new LocalReleaseProvider(releaseDirectory, githubRepository) - new EngineReleaseProvider(mergedRepository) - } } diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/graalvm/GraalCEReleaseProvider.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/graalvm/GraalCEReleaseProvider.scala index 3da25c71f6..8e758f0afd 100644 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/graalvm/GraalCEReleaseProvider.scala +++ b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/graalvm/GraalCEReleaseProvider.scala @@ -6,7 +6,6 @@ import org.enso.cli.task.TaskProgress import org.enso.runtimeversionmanager.OS import org.enso.runtimeversionmanager.components.GraalVMVersion import org.enso.runtimeversionmanager.releases.github.GithubReleaseProvider -import org.enso.runtimeversionmanager.releases.local.LocalReleaseProvider import org.enso.runtimeversionmanager.releases.{ ReleaseProviderException, SimpleReleaseProvider @@ -59,17 +58,6 @@ object GraalCEReleaseProvider { */ val default = new GraalCEReleaseProvider(githubRepository) - /** Creates a GraalVM provider that uses a local repository first, falling - * back to the default one. - */ - def fromLocalRepository( - releaseDirectory: Path - ): GraalVMRuntimeReleaseProvider = { - val mergedRepository = - new LocalReleaseProvider(releaseDirectory, githubRepository) - new GraalCEReleaseProvider(mergedRepository) - } - /** Generates the name of the package for the currently running OS and a * specified release version. */ diff --git a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/local/LocalReleaseProvider.scala b/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/local/LocalReleaseProvider.scala deleted file mode 100644 index 08391f0f0b..0000000000 --- a/lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/releases/local/LocalReleaseProvider.scala +++ /dev/null @@ -1,112 +0,0 @@ -package org.enso.runtimeversionmanager.releases.local - -import java.nio.file.Path - -import cats.syntax.traverse._ -import com.typesafe.scalalogging.Logger -import org.enso.cli.task.TaskProgress -import org.enso.runtimeversionmanager.FileSystem -import org.enso.runtimeversionmanager.releases.{ - Asset, - Release, - SimpleReleaseProvider -} - -import scala.io.Source -import scala.util.{Try, Using} - -/** A [[SimpleReleaseProvider]] that uses a repository located on a local file - * system as its primary source and falls back to some other specified - * repository in other cases. - * - * It can be used to implement bundling some versions with an installer - it - * can point to this local repository, so that any required bundled versions - * are installed from the bundle and any other versions are handled using the - * default repository. - * - * It is given a `releaseDirectory` that should contain separate directories - * for each local release. The name of each subdirectory corresponds to its - * release tag and every file in that subdirectory is considered as an asset of - * that release. - * - * It loads the list of releases at construction and thus may throw an error if - * it cannot access the provided directory. - */ -class LocalReleaseProvider( - releaseDirectory: Path, - fallback: SimpleReleaseProvider -) extends SimpleReleaseProvider { - private val logger = Logger[LocalReleaseProvider] - private val localDirectories: Seq[Path] = - FileSystem.listDirectory(releaseDirectory).filter { dir => - val isIgnoredFile = - FileSystem.ignoredFileNames.contains(dir.getFileName.toString) - !isIgnoredFile - } - - /** @inheritdoc */ - override def releaseForTag(tag: String): Try[Release] = { - val localPath = localDirectories.find(_.getFileName.toString == tag) - localPath - .map(wrapLocalDirectory) - .getOrElse { fallback.releaseForTag(tag) } - } - - /** @inheritdoc */ - override def listReleases(): Try[Seq[Release]] = { - val remote = fallback - .listReleases() - .recover { error => - logger.warn( - s"The remote release provider failed with $error, but " + - s"locally bundled releases are available." - ) - Seq.empty - } - .get - findLocalReleases() map { local => - val localTags = local.map(_.tag).toSet - val remoteDeduplicated = remote.filter { remoteRelease => - val hasLocalCorrespondent = localTags.contains(remoteRelease.tag) - !hasLocalCorrespondent - } - local ++ remoteDeduplicated - } - } - - /** An asset that is on the local filesystem. */ - private case class LocalAsset(assetPath: Path) extends Asset { - - /** @inheritdoc */ - override def fileName: String = assetPath.getFileName.toString - - /** @inheritdoc */ - override def downloadTo(path: Path): TaskProgress[Unit] = - TaskProgress.runImmediately { - FileSystem.copyFile(assetPath, path) - } - - /** @inheritdoc */ - override def fetchAsText(): TaskProgress[String] = - TaskProgress.fromTry { - Using(Source.fromFile(assetPath.toFile)) { src => - src.getLines().mkString("\n") - } - } - } - - private case class LocalRelease( - override val tag: String, - override val assets: Seq[LocalAsset] - ) extends Release - - /** Creates a [[LocalRelease]] defined by a local directory. */ - private def wrapLocalDirectory(path: Path): Try[Release] = Try { - val tag = path.getFileName.toString - val assets = FileSystem.listDirectory(path).map(LocalAsset) - LocalRelease(tag, assets) - } - - private def findLocalReleases(): Try[Seq[Release]] = - localDirectories.map(wrapLocalDirectory).toList.sequence[Try, Release] -} diff --git a/project/DistributionPackage.scala b/project/DistributionPackage.scala index bd37564c08..0445b1cb73 100644 --- a/project/DistributionPackage.scala +++ b/project/DistributionPackage.scala @@ -1,7 +1,10 @@ -import sbt.{file, singleFileFinder, File, IO} +import sbt.internal.util.ManagedLogger +import sbt._ import sbt.io.syntax.fileToRichFile import sbt.util.{CacheStore, CacheStoreFactory, FileInfo, Tracked} +import scala.sys.process._ + object DistributionPackage { def copyDirectoryIncremental( source: File, @@ -144,4 +147,315 @@ object DistributionPackage { cacheFactory.make("launcher-rootfiles") ) } + + sealed trait OS { + def name: String + def graalName: String = name + def executableName(base: String): String = base + def archiveExt: String = ".tar.gz" + def isUNIX: Boolean = true + } + object OS { + case object Linux extends OS { + override def name: String = "linux" + } + case object MacOS extends OS { + override def name: String = "macos" + override def graalName: String = "darwin" + } + case object Windows extends OS { + override def name: String = "windows" + override def executableName(base: String): String = base + ".exe" + override def archiveExt: String = ".zip" + override def isUNIX: Boolean = false + } + + val platforms = Seq(Linux, MacOS, Windows) + } + + sealed trait Architecture { + def name: String + } + object Architecture { + case object X64 extends Architecture { + override def name: String = "amd64" + } + + val archs = Seq(X64) + } + + /** A helper class that manages building distribution artifacts. */ + class Builder( + ensoVersion: String, + graalVersion: String, + graalJavaVersion: String, + artifactRoot: File + ) { + + def artifactName( + component: String, + os: OS, + architecture: Architecture + ): String = + s"enso-$component-$ensoVersion-${os.name}-${architecture.name}" + + def graalInPackageName: String = + s"graalvm-ce-java$graalJavaVersion-$graalVersion" + + private def extractZip(archive: File, root: File): Unit = { + IO.createDirectory(root) + val exitCode = Process( + Seq("unzip", "-q", archive.toPath.toAbsolutePath.normalize.toString), + cwd = Some(root) + ).! + if (exitCode != 0) { + throw new RuntimeException(s"Cannot extract $archive.") + } + } + + private def extractTarGz(archive: File, root: File): Unit = { + IO.createDirectory(root) + val exitCode = Process( + Seq( + "tar", + "xf", + archive.toPath.toAbsolutePath.toString + ), + cwd = Some(root) + ).! + if (exitCode != 0) { + throw new RuntimeException(s"Cannot extract $archive.") + } + } + + private def extract(archive: File, root: File): Unit = { + if (archive.getName.endsWith("zip")) { + extractZip(archive, root) + } else { + extractTarGz(archive, root) + } + } + + def copyGraal( + log: ManagedLogger, + os: OS, + architecture: Architecture, + runtimeDir: File + ): Unit = { + val packageName = s"graalvm-${os.name}-${architecture.name}-" + + s"$graalVersion-$graalJavaVersion" + val root = artifactRoot / packageName + if (!root.exists()) { + log.info( + s"Downloading GraalVM $graalVersion Java $graalJavaVersion " + + s"for $os $architecture" + ) + val graalUrl = + s"https://github.com/graalvm/graalvm-ce-builds/releases/download/" + + s"vm-$graalVersion/" + + s"graalvm-ce-java$graalJavaVersion-${os.graalName}-" + + s"${architecture.name}-$graalVersion${os.archiveExt}" + val archive = artifactRoot / (packageName + os.archiveExt) + val exitCode = (url(graalUrl) #> archive).! + if (exitCode != 0) { + throw new RuntimeException(s"Graal download from $graalUrl failed.") + } + + extract(archive, root) + } + + IO.copyDirectory( + root / graalInPackageName, + runtimeDir / graalInPackageName + ) + } + + def copyEngine(os: OS, architecture: Architecture, distDir: File): Unit = { + val engine = builtArtifact("engine", os, architecture) + if (!engine.exists()) { + throw new IllegalStateException( + s"Cannot create bundle for $os / $architecture because corresponding " + + s"engine has not been built." + ) + } + + IO.copyDirectory(engine / s"enso-$ensoVersion", distDir / ensoVersion) + } + + def makeExecutable(file: File): Unit = { + val ownerOnly = false + file.setExecutable(true, ownerOnly) + } + + def fixLauncher(root: File, os: OS): Unit = { + makeExecutable(root / "enso" / "bin" / os.executableName("enso")) + IO.createDirectories( + Seq("dist", "config", "runtime").map(root / "enso" / _) + ) + } + + def makeArchive(root: File, rootDir: String, target: File): Unit = { + val exitCode = if (target.getName.endsWith("zip")) { + Process( + Seq( + "zip", + "-q", + "-r", + target.toPath.toAbsolutePath.normalize.toString, + rootDir + ), + cwd = Some(root) + ).! + } else { + Process( + Seq( + "tar", + "-czf", + target.toPath.toAbsolutePath.normalize.toString, + rootDir + ), + cwd = Some(root) + ).! + } + if (exitCode != 0) { + throw new RuntimeException(s"Failed to create archive $target") + } + } + + /** Path to an arbitrary built artifact. */ + def builtArtifact( + component: String, + os: OS, + architecture: Architecture + ): File = artifactRoot / artifactName(component, os, architecture) + + /** Path to the artifact that is built on this local machine. */ + def localArtifact(component: String): File = { + val architecture = Architecture.X64 + val os = + if (Platform.isWindows) OS.Windows + else if (Platform.isLinux) OS.Linux + else if (Platform.isMacOS) OS.MacOS + else throw new IllegalStateException("Unknown OS") + artifactRoot / artifactName(component, os, architecture) + } + + /** Path to a built archive. + * + * These archives are built by [[makePackages]] and [[makeBundles]]. + */ + def builtArchive( + component: String, + os: OS, + architecture: Architecture + ): File = + artifactRoot / (artifactName( + component, + os, + architecture + ) + os.archiveExt) + + /** Creates compressed and ready for release packages for the launcher and + * engine. + * + * A project manager package is not created, as we release only its bundle. + * See [[makeBundles]]. + * + * It does not trigger any builds. Instead, it uses available artifacts + * placed in `artifactRoot`. These artifacts may be created using the + * `enso/build*Distribution` tasks or they may come from other workers (as + * is the case in the release CI where the artifacts are downloaded from + * other jobs). + */ + def makePackages = Command.command("makePackages") { state => + val log = state.log + for { + os <- OS.platforms + arch <- Architecture.archs + } { + val launcher = builtArtifact("launcher", os, arch) + if (launcher.exists()) { + fixLauncher(launcher, os) + val archive = builtArchive("launcher", os, arch) + makeArchive(launcher, "enso", archive) + log.info(s"Created $archive") + } + + val engine = builtArtifact("engine", os, arch) + if (engine.exists()) { + if (os.isUNIX) { + makeExecutable(engine / s"enso-$ensoVersion" / "bin" / "enso") + } + val archive = builtArchive("engine", os, arch) + makeArchive(engine, s"enso-$ensoVersion", archive) + log.info(s"Created $archive") + } + } + state + } + + private def cleanDirectory(dir: File): Unit = { + for (f <- IO.listFiles(dir)) { + IO.delete(f) + } + } + + /** Creates launcher and project-manager bundles that include the component + * itself, the engine and a Graal runtime. + * + * It will download the GraalVM runtime and cache it in `artifactRoot` so + * further invocations for the same version will not need to download it. + * + * It does not trigger any builds. Instead, it uses available artifacts + * placed in `artifactRoot`. These artifacts may be created using the + * `enso/build*Distribution` tasks or they may come from other workers (as + * is the case in the release CI where the artifacts are downloaded from + * other jobs). + */ + def makeBundles = Command.command("makeBundles") { state => + val log = state.log + for { + os <- OS.platforms + arch <- Architecture.archs + } { + val launcher = builtArtifact("launcher", os, arch) + if (launcher.exists()) { + fixLauncher(launcher, os) + copyEngine(os, arch, launcher / "enso" / "dist") + copyGraal(log, os, arch, launcher / "enso" / "runtime") + + val archive = builtArchive("bundle", os, arch) + makeArchive(launcher, "enso", archive) + + cleanDirectory(launcher / "enso" / "dist") + cleanDirectory(launcher / "enso" / "runtime") + + log.info(s"Created $archive") + } + + val pm = builtArtifact("project-manager", os, arch) + if (pm.exists()) { + if (os.isUNIX) { + makeExecutable(pm / "enso" / "bin" / "project-manager") + } + + copyEngine(os, arch, pm / "enso" / "dist") + copyGraal(log, os, arch, pm / "enso" / "runtime") + IO.copyFile( + file("distribution/enso.bundle.template"), + pm / "enso" / ".enso.bundle" + ) + + val archive = builtArchive("project-manager", os, arch) + makeArchive(pm, "enso", archive) + + cleanDirectory(pm / "enso" / "dist") + cleanDirectory(pm / "enso" / "runtime") + + log.info(s"Created $archive") + } + } + state + } + } } diff --git a/tools/ci/prepare-distribution-env.sh b/tools/ci/prepare-distribution-env.sh index fc5c1663c3..f3bc3e57d7 100644 --- a/tools/ci/prepare-distribution-env.sh +++ b/tools/ci/prepare-distribution-env.sh @@ -1,14 +1,21 @@ #!/usr/bin/env bash DIST_ARCH=amd64 -LAUNCHER_DIST_ROOT=enso-launcher-$DIST_VERSION-$DIST_OS-$DIST_ARCH +BUILD_ROOT=built-distribution +LAUNCHER_DIST_NAME=enso-launcher-$DIST_VERSION-$DIST_OS-$DIST_ARCH +LAUNCHER_DIST_ROOT=$BUILD_ROOT/$LAUNCHER_DIST_NAME LAUNCHER_DIST_DIR=$LAUNCHER_DIST_ROOT/enso -ENGINE_DIST_ROOT=enso-engine-$DIST_VERSION-$DIST_OS-$DIST_ARCH +ENGINE_DIST_NAME=enso-engine-$DIST_VERSION-$DIST_OS-$DIST_ARCH +ENGINE_DIST_ROOT=$BUILD_ROOT/$ENGINE_DIST_NAME ENGINE_DIST_DIR=$ENGINE_DIST_ROOT/enso-$DIST_VERSION -PROJECTMANAGER_DIST_ROOT=enso-project-manager-$DIST_VERSION-$DIST_OS-$DIST_ARCH +PROJECTMANAGER_DIST_NAME=enso-project-manager-$DIST_VERSION-$DIST_OS-$DIST_ARCH +PROJECTMANAGER_DIST_ROOT=$BUILD_ROOT/$PROJECTMANAGER_DIST_NAME PROJECTMANAGER_DIST_DIR=$PROJECTMANAGER_DIST_ROOT/enso +echo "LAUNCHER_DIST_NAME=$LAUNCHER_DIST_NAME" >> $GITHUB_ENV echo "LAUNCHER_DIST_DIR=$LAUNCHER_DIST_DIR" >> $GITHUB_ENV echo "LAUNCHER_DIST_ROOT=$LAUNCHER_DIST_ROOT" >> $GITHUB_ENV +echo "ENGINE_DIST_NAME=$ENGINE_DIST_NAME" >> $GITHUB_ENV echo "ENGINE_DIST_DIR=$ENGINE_DIST_DIR" >> $GITHUB_ENV echo "ENGINE_DIST_ROOT=$ENGINE_DIST_ROOT" >> $GITHUB_ENV +echo "PROJECTMANAGER_DIST_NAME=$PROJECTMANAGER_DIST_NAME" >> $GITHUB_ENV echo "PROJECTMANAGER_DIST_DIR=$PROJECTMANAGER_DIST_DIR" >> $GITHUB_ENV echo "PROJECTMANAGER_DIST_ROOT=$PROJECTMANAGER_DIST_ROOT" >> $GITHUB_ENV