Update Documentation and Build Settings (#993)

This commit is contained in:
Radosław Waśko 2020-07-21 10:14:26 +02:00 committed by GitHub
parent 239a5f18e9
commit 1a38f7c331
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 252 additions and 88 deletions

View File

@ -96,10 +96,10 @@ jobs:
- name: Build the Uberjar
run: sbt -no-colors runner/assembly
- name: Test the Uberjar
run: ./enso.jar --run tools/ci/Test.enso
run: ./runner.jar --run tools/ci/Test.enso
- name: Test the Repl in the Uberjar
shell: bash
run: cat tools/ci/artifact-test/repl/ReplTest.in | ./enso.jar --repl | diff -q tools/ci/artifact-test/repl/ReplTest.out - && echo "Test successful"
run: cat tools/ci/artifact-test/repl/ReplTest.in | ./runner.jar --repl | diff -q tools/ci/artifact-test/repl/ReplTest.out - && echo "Test successful"
# This job is responsible for building the artifacts
build:
@ -152,29 +152,30 @@ jobs:
run: sbt --no-colors runner/assembly
- name: Build the Project Manager Uberjar
run: sbt --no-colors project-manager/assembly
- name: Build the Manifest
run: |
cp distribution/manifest.template.yaml manifest.yaml
echo "graal-vm-version: $graalVersion" >> manifest.yaml
echo "graal-java-version: $javaVersion" >> manifest.yaml
- name: Prepare Distribution
run: |
DIST_VERSION=$(./enso.jar --version --json | jq -r '.version')
DIST_VERSION=$(./runner.jar --version --json | jq -r '.version')
DIST_ROOT=enso-engine-$DIST_VERSION
DIST_DIR=$DIST_ROOT/enso-$DIST_VERSION
mkdir -p $DIST_DIR
mkdir $DIST_DIR/component
cp runtime.jar $DIST_DIR/component
mv enso.jar $DIST_DIR/component
mv runner.jar $DIST_DIR/component
mv project-manager.jar $DIST_DIR/component
cp -r distribution/std-lib $DIST_DIR/std-lib
cp -r distribution/bin $DIST_DIR/bin
chmod +x $DIST_DIR/bin/enso
chmod +x $DIST_DIR/bin/project-manager
cp manifest.yaml $DIST_DIR
echo ::set-env name=DIST_ROOT::$DIST_ROOT
echo ::set-env name=DIST_DIR::$DIST_DIR
- name: Build the Parser JS Bundle
run: sbt -no-colors syntaxJS/fullOptJS
- name: Build the Manifest
run: |
cp distribution/manifest.template.yml manifest.yml
echo "graal-vm-version: $graalVersion" >> manifest.yml
echo "graal-java-version: $javaVersion" >> manifest.yml
# Publish
- name: Publish the Distribution Artifact
@ -205,7 +206,7 @@ jobs:
uses: actions/upload-artifact@v2
with:
name: manifest
path: manifest.yml
path: manifest.yaml
- name: Prepare AWS Session
run: |
aws configure --profile s3-upload <<-EOF > /dev/null 2>&1
@ -314,7 +315,7 @@ jobs:
mkdir -p ${{ env.DIST_DIR }}
mkdir ${{ env.DIST_DIR }}/bin
mkdir ${{ env.DIST_DIR }}/dist
mkdir ${{ env.DIST_DIR }}/jvm
mkdir ${{ env.DIST_DIR }}/runtime
cp distribution/launcher/README.md ${{ env.DIST_DIR }}
cp distribution/launcher/NOTICE ${{ env.DIST_DIR }}
cp -r distribution/launcher/components-licences ${{ env.DIST_DIR }}

View File

@ -903,9 +903,9 @@ lazy val runner = project
},
mainClass in (Compile, run) := Some("org.enso.runner.Main"),
mainClass in assembly := (Compile / run / mainClass).value,
assemblyJarName in assembly := "enso.jar",
assemblyJarName in assembly := "runner.jar",
test in assembly := {},
assemblyOutputPath in assembly := file("enso.jar"),
assemblyOutputPath in assembly := file("runner.jar"),
assemblyMergeStrategy in assembly := {
case PathList("META-INF", file, xs @ _*) if file.endsWith(".DSA") =>
MergeStrategy.discard

View File

@ -1,3 +1,3 @@
COMP_PATH=$(dirname "$0")/../component
exec java -jar -Dtruffle.class.path.append="$COMP_PATH/runtime.jar" -Dpolyglot.engine.IterativePartialEscape=true $JAVA_OPTS $COMP_PATH/enso.jar "$@"
exec java -jar -Dtruffle.class.path.append="$COMP_PATH/runtime.jar" -Dpolyglot.engine.IterativePartialEscape=true $JAVA_OPTS $COMP_PATH/runner.jar "$@"
exit

View File

@ -1,3 +1,3 @@
set comp-dir=%~dp0\..\component
java -jar -Dtruffle.class.path.append=%comp-dir%\runtime.jar -Dpolyglot.engine.IterativePartialEscape=true %JAVA_OPTS% %comp-dir%\enso.jar %*
java -jar -Dtruffle.class.path.append=%comp-dir%\runtime.jar -Dpolyglot.engine.IterativePartialEscape=true %JAVA_OPTS% %comp-dir%\runner.jar %*
exit /B %errorlevel%

View File

@ -233,12 +233,12 @@ inside the `runner` subproject:
sbt "runner/assembly"
```
This will produce an executable `enso.jar` fat jar in the repository root.
It's self contained, with its only dependencies being available inside
a vanilla GraalVM distribution. To run it, use:
This will produce an executable `runner.jar` fat jar and a `runtime.jar` fat jar
in the repository root. The `runner.jar` depends only on the `runtime.jar` and a
vanilla GraalVM distribution. To run it, use:
```bash
JAVA_HOME=<PATH_TO_GRAAL_HOME> ./enso.jar <CLI_ARGS>
JAVA_HOME=<PATH_TO_GRAAL_HOME> ./runner.jar <CLI_ARGS>
```
#### Building the Launcher Native Binary
@ -267,7 +267,7 @@ To run it:
1. Build (or download from the CI server) the CLI Fat Jar.
2. Fill in the `engine/language-server/jupyter-kernel/enso/kernel.json`
file, providing correct paths to the `enso.jar` distribution and GraalVM
file, providing correct paths to the `runner.jar` distribution and GraalVM
`JAVA_HOME`.
3. Run:
@ -433,7 +433,7 @@ Below are options uses by the Language Server:
To run the Language Server on 127.0.0.1:8080 type:
```bash
./enso.jar \
./runner.jar \
--server \
--root-id 3256d10d-45be-45b1-9ea4-7912ef4226b1 \
--path /tmp/content-root

View File

@ -12,36 +12,87 @@ be structured and how it should behave.
<!-- MarkdownTOC levels="2,3" autolink="true" -->
- [Enso Home Layout](#enso-home-layout)
- [Universal Launcher Script](#universal-launcher-script)
- [Universal Launcher](#universal-launcher)
- [Enso Distribution Layout](#enso-distribution-layout)
- [Portable Enso Distribution Layout](#portable-enso-distribution-layout)
- [Installed Enso Distribution Layout](#installed-enso-distribution-layout)
- [Installing from a Portable Distribution](#installing-from-a-portable-distribution)
- [Layout of an Enso Version Package](#layout-of-an-enso-version-package)
- [Standard Library](#standard-library)
- [Resolvers](#resolvers)
<!-- /MarkdownTOC -->
## Enso Home Layout
All of Enso's binaries, packages, etc. are installed into a directory in
the user's home directory. For macOS and linux distributions that's `~/.enso`,
by default. The distribution is fully portable, so it never makes any
assumptions about actually being placed in any particular directory.
## Universal Launcher
The [universal launcher](./launcher.md) should be able to launch the
proper version of Enso executable based on the version specified in the project
being run, or use the default version if none specified. It should also be able
to launch other Enso components, provided as
[plugins](./launcher.md#running-plugins).
> This launcher is under development. Until it is in a ready-to-use state, the
> Enso version packages provide simple launcher scripts in the `bin` directory
> of that package. They are a temporary replacement for the launcher
> functionality, so once the universal launcher matures, they will be removed.
> The universal launcher will not call the components through these scripts, as
> it must have full control over which JVM is chosen and its parameters.
## Enso Distribution Layout
Enso is distributed as a portable package that can be extracted anywhere on the
system and run. It can also be installed for the local user into system-defined
directories, as explained below.
### Portable Enso Distribution Layout
All files in the directory structure, except for configuration, can be safely
removed and the launcher will re-download them if needed.
The directory structure is as follows:
```
install-location
extraction-location
├── bin
│ └── enso # The universal launcher script, responsible for choosing the appropriate compiler version.
│ └── enso # The universal launcher, responsible for choosing the appropriate compiler version (TODO [RW] it may be stored in a different place).
├── config
│ └── global-config.yaml # Global user configuration.
├── dist # Per-compiler-version distribution directories.
│ ├── default -> enso-1.2.0 # A symlink to the version that should be used when no version is explicitly specified.
│ ├── enso-1.0.0 # A full distribution of given Enso version, described below.
│ ├── 1.0.0 # A full distribution of given Enso version, described below.
│ │ └── <truncated>
│ └── enso-1.2.0 # A full distribution of given Enso version, described below.
│ └── 1.2.0 # A full distribution of given Enso version, described below.
│ └── <truncated>
├── jvm # A directory storing (optional) distributions of the JVM used by the Enso distributions.
├── runtime # A directory storing distributions of the JVM used by the Enso distributions.
│ └── graalvm-ce-27.1.1
└── cache # A directory storing downloaded libraries and resolvers. Can be removed safely.
├── libraries # Contains downloaded libraries.
├── lib
│ └── src # Contains sources of downloaded libraries.
│ └── Dataframe # Each library may be stored in multiple version.
│ └── 1.7.0 # Each version contains a standard Enso package.
│ ├── package.yaml
│ └── src
│ ├── List.enso
│ ├── Number.enso
│ └── Text.enso
├── resolvers # Contains resolver specifications, described below.
│ ├── lts-1.56.7.yaml
│ └── lts-2.0.8.yaml
├── README.md # Information on layout and usage of the Enso distribution.
├── .enso.portable # An empty file that allows the universal launcher to detect that if it is run from this directory, it should run in portable distribution mode.
├── NOTICE # A copyright notice regarding components that are included in the distribution of the universal launcher.
└── components-licences # Contains licences of distributed components, as described in the NOTICE.
```
### Installed Enso Distribution Layout
After installation, the directory structure is following:
```
ENSO_DATA_DIRECTORY
├── dist # Per-compiler-version distribution directories.
│ ├── 1.0.0 # A full distribution of given Enso version, described below.
│ │ └── <truncated>
│ └── 1.2.0 # A full distribution of given Enso version, described below.
│ └── <truncated>
├── runtime # A directory storing (optional) distributions of the JVM used by the Enso distributions.
│ └── graalvm-ce-27.1.1
├── lib
│ └── src # Contains sources of downloaded libraries.
│ └── Dataframe # Each library may be stored in multiple version.
│ └── 1.7.0 # Each version contains a standard Enso package.
│ ├── package.yaml
@ -52,12 +103,41 @@ install-location
└── resolvers # Contains resolver specifications, described below.
├── lts-1.56.7.yaml
└── lts-2.0.8.yaml
ENSO_CONFIG_DIRECTORY
└── global-config.yaml # Global user configuration.
ENSO_BIN_DIRECTORY
└── enso # The universal launcher, responsible for choosing the appropriate compiler version (TODO [RW] it may be stored in a different place).
```
## Universal Launcher Script
The universal launcher script should be able to launch the proper version of
Enso executable based on the version specified in the project being run,
or use the default version if none specified.
Where `ENSO_DATA_DIRECTORY`, `ENSO_CONFIG_DIRECTORY` and `ENSO_BIN_DIRECTORY`
are environment variables that define the directory structure. They can be used
to override placement of the components listed above. However, most of the time
they do not have to be set, as they use system-specific defaults.
If not set, each of these three environment variables defaults to the following
value, depending on the system:
| | Linux | macOS | Windows |
|-------------------------|--------------------------------------------------------------------|-----------------------------------------------|------------------------------|
| `ENSO_DATA_DIRECTORY` | `$XDG_DATA_HOME/enso/` which defaults to `$HOME/.local/share/enso` | `$HOME/Library/Application Support/org.enso/` | `%LocalAppData%/enso` |
| `ENSO_CONFIG_DIRECTORY` | `$XDG_CONFIG_HOME/enso/` which defaults to `$HOME/.config/enso` | `$HOME/Library/Preferences/org.enso/` | `%LocalAppData%/enso/config` |
| `ENSO_BIN_DIRECTORY` | `$XDG_BIN_HOME` which defaults to `$HOME/.local/bin` | `$HOME/.local/bin` | `%LocalAppData%/enso/bin` |
### Installing from a Portable Distribution
After downloading and extracting the portable distribution, the user can run
`extraction-location/bin/enso install distribution` to install it locally. This
will copy the files from the portable distribution into the installed locations
which are described above and then remove the original files.
On Linux and macOS, if `ENSO_BIN_DIRECTORY` (`~/.local/bin` by default) is not
on system `PATH`, the installer will issue a warning, telling the user how they
can add it. On Windows, the installer automatically adds `ENSO_BIN_DIRECTORY` to
the user's `PATH`.
The installed distribution can be removed by running
`enso uninstall distribution`.
## Layout of an Enso Version Package
This section describes the structure of a single version distribution. This
@ -71,9 +151,11 @@ The layout of such a distribution is as follows:
```
enso-1.0.0
├── manifest.yaml # A manifest file defining metadata about this Enso version.
├── component # Contains all the executable tools and their dependencies.
│ └── enso.jar # The main executable of the distribution. CLI entry point.
└── std-lib # Contains all the pre-installed libraries compiler version.
│ ├── runner.jar # The main executable of the distribution. CLI entry point.
│ └── runtime.jar # The language runtime. It is loaded by other JVM components, like the runner.
└── std-lib # Contains all the libraries that are pre-installed within that compiler version.
├── Http # Every version sub-directory is just an Enso package containing the library.
│ ├── package.yaml
│ ├── polyglot

View File

@ -19,8 +19,10 @@ This document describes available command-line options of the Enso launcher.
- [Commands](#commands)
- [`new`](#new)
- [`install`](#install)
- [`uninstall`](#uninstall)
- [`install engine`](#install-engine)
- [`uninstall engine`](#uninstall-engine)
- [`install distribution`](#install-distribution)
- [`uninstall distribution`](#uninstall-distribution)
- [`list`](#list)
- [`default`](#default)
- [`config`](#config)
@ -33,6 +35,7 @@ This document describes available command-line options of the Enso launcher.
- [General Options](#general-options)
- [`--version`](#--version)
- [`--use-system-jvm`](#--use-system-jvm)
- [JVM Options](#jvm-options)
<!-- /MarkdownTOC -->
@ -52,20 +55,52 @@ Examples:
# creates the project in the current directory, using the 2.0.1 version
```
### `install`
Installs a specific version of Enso.
### `install engine`
Installs a specific version of the Enso engine.
Examples:
```bash
> enso install 2.0.1
> enso install engine 2.0.1
```
### `uninstall`
Uninstalls a specific version of Enso.
### `uninstall engine`
Uninstalls a specific version of the Enso engine.
Examples:
```bash
> enso uninstall 2.0.1
> enso uninstall engine 2.0.1
```
### `install distribution`
Installs a portable Enso distribution into system-defined directories, as
explained in
[Installed Enso Distribution Layout](./distribution.md#installed-enso-distribution-layout).
By default, it asks the user for confirmation, but this can be skipped by
adding a `--yes` flag.
Examples:
```
> extraction-location/bin/enso install distribution
This will install Enso to ~/.local/share/enso/.
Configuration will be placed in ~/.config/enso/.
The universal `enso` launcher will be placed in ~/.local/bin/.
Do you want to continue? [Y/n]
```
### `uninstall distribution`
Uninstalls an installed Enso distribution from the installation location
described in
[Installed Enso Distribution Layout](./distribution.md#installed-enso-distribution-layout).
It removes the universal launcher and all components. By default, it asks the
user for confirmation, but this can be skipped by adding a `--yes` flag.
Examples:
```
> enso uninstall distribution
This will completely uninstall Enso from ~/.local/share/enso/,
remove configuration from ~/.config/enso/
and the launcher script from ~/.local/bin/.
Do you want to continue? [y/N]
```
### `list`
@ -196,3 +231,12 @@ command.
### `--use-system-jvm`
Tells the launcher to use the default JVM (based on `JAVA_HOME`) instead of the
managed one. Will not work if the set-up JVM version is not GraalVM.
## JVM Options
For commands that launch an Enso component inside a JVM (`repl`, `run` and
`language-server`), additional parameters are passed to the launcher components.
Moreover, it is possible to pass parameters to the JVM that is used to launch
these components, which may be helpful with debugging.
A parameter of the form `--jvm.argumentName=argumentValue` will be passed to the
JVM as `-DargumentName=argumentValue`.

View File

@ -14,6 +14,8 @@ command-line interface is described in the [CLI](./launcher-cli.md) document.
<!-- MarkdownTOC levels="2,3" autolink="true" -->
- [Launcher Distribution](#launcher-distribution)
- [Using Multiple Launcher Versions Side-By-Side](#using-multiple-launcher-versions-side-by-side)
- [Detecting Portable Distribution](#detecting-portable-distribution)
- [Launcher Build](#launcher-build)
- [Portability](#portability)
- [Project Management](#project-management)
@ -25,6 +27,7 @@ command-line interface is described in the [CLI](./launcher-cli.md) document.
- [Downloading Enso Releases](#downloading-enso-releases)
- [Downloading GraalVM Releases](#downloading-graalvm-releases)
- [Running Enso Components](#running-enso-components)
- [Running Plugins](#running-plugins)
- [Global User Configuration](#global-user-configuration)
- [Updating the Launcher](#updating-the-launcher)
- [Minimal Required Launcher Version](#minimal-required-launcher-version)
@ -34,17 +37,27 @@ command-line interface is described in the [CLI](./launcher-cli.md) document.
## Launcher Distribution
The launcher is distributed as a native binary for each platform (Windows,
Linux, Mac). It is distributed in a ZIP archive as described in
[Enso Home Layout](./distribution.md#enso-home-layout), except that the
component directories are empty (they will be populated when Enso versions are
downloaded by the launcher on-demand). The only non-empty directory is `bin`
which is where the launcher binary is placed. The ZIP file should also contain a
[README](../../distribution/launcher/README.md) describing the distribution and
basic usage.
Linux, macOS). It is distributed in a ZIP archive as described in
[Enso Distribution Layout](./distribution.md#enso-distribution-layout) in two
flavors - as packages containing just the launcher binary that can then
download and install desired versions of the engine and as bundles that already
contain the latest version of Enso engine and Graal runtime corresponding to it.
The distribution structure is portable - this directory structure can be placed
anywhere in the system. The launcher places all components in this directory
structure by accessing them as relative to the location of its binary.
### Using Multiple Launcher Versions Side-By-Side
Multiple portable distributions of the launcher can be used side-by-side. To use
multiple installed distributions, some tricks are necessary - before launching a
different version, the environment variables `ENSO_DATA_DIRECTORY`,
`ENSO_CONFIG_DIRECTORY` and `ENSO_BIN_DIRECTORY` have to be set to directories
corresponding to that version.
### Detecting Portable Distribution
As described in
[Enso Distribution Layout](./distribution.md#enso-distribution-layout), the
launcher can either be run in a portable distribution or installed locally. The
launcher must detect if its run as the portable or installed distribution. When
run, the launcher checks if it is placed in a directory called `bin` and checks
the parent directory for a file called `.enso.portable`. If such file is found,
the launcher runs in portable mode. Otherwise, it runs in installed mode.
## Launcher Build
The launcher is built using
@ -56,7 +69,7 @@ small and fast launching executable.
On Linux, it is possible to statically link all libraries required by the Native
Image, thus ensuring portability between Linux distributions.
On Windows and Mac, it is not possible to statically link against system
On Windows and macOS, it is not possible to statically link against system
libraries, but this should not hinder portability as the system libraries are
generally compatible between distribution versions on these platforms.
Non-system dependencies are included in the binary on these platforms as well.
@ -151,7 +164,22 @@ parameters and configuration:
used.
Additional arguments passed to the launcher are forwarded to the launched
component.
component. Moreover, options for the JVM that is used to run the components
can also be provided, as described in
[JVM Options](./launcher-cli.md#jvm-options).
### Running Plugins
If the launcher gets an unknown command `foo`, it tries to run `enso-foo` and
pass all the arguments that follow. If `enso-foo` is not found, it fails as
normal. This can be used to implement plugins that are launched through the
universal launcher. For example, the Enso IDE can provide an `enso-ide`
executable, allowing users to launch the IDE by typing `enso ide`.
For a plugin to be recognized by the launcher, it needs to support a
`--synopsis` option - running `enso-foo --synopsis` should print a short
description and return with exit code 0, for the plugin to be considered
supported. That description will be included in the command listing printed by
`enso help`.
## Global User Configuration
The launcher allows to edit global user configuration, saved in the `config`

View File

@ -79,7 +79,8 @@ specified on a per-language basis, in the
### The `package.yaml` File
`package.yaml` describes certain package metadata, such as its name, authors
and version. It also includes the list of dependencies of the package.
and version. It also includes the list of extra dependencies of the package
(dependencies that are not present in the resolver or need a version override).
The following is an example of this manifest file.
```yaml
@ -97,14 +98,21 @@ extra-dependencies:
```
The following is the specification of the manifest fields.
Fields marked as **Optional (required for publishing)** are completely optional
during development - if not specified, their default values will be used.
However, they must be specified before publishing the package. A package missing
any of these fields cannot be published.
#### license
**Optional** *String*: The short license name of this package. Defaults to
`None`, meaning the package is not safe for use by third parties.
**Optional (required for publishing)** *String*: The short license name of this
package. Defaults to `None`, meaning the package is not safe for use by third
parties.
#### version
**Required** *String*: The [semantic versioning](https://semver.org/) string,
in the `major.minor.patch` format.
**Optional (required for publishing)** *String*: The
[semantic versioning](https://semver.org/) string, in the `major.minor.patch`
format. If not set, it defaults to `dev` (which can be used for development, but
is not a valid version for publishing).
#### author
**Optional** *String* or *List of Strings*: The name(s) and contact info(s) of
@ -117,8 +125,9 @@ format.
#### resolver
**Note** This field is not currently implemented.
**Required** *String*: The resolver name, used to choose compiler version and
basic libraries set.
**Optional (required for publishing)** *String*: The resolver name, used to
choose compiler version and basic libraries set. If not set, the system-default
resolver will be used.
> The actionables for this section are:
>

View File

@ -86,8 +86,9 @@ A release is considered _official_ once it has been made into a release on
may not be changed in any way, except to mark it as broken.
#### Manifest File
Each GitHub release contains an asset named `manifest.yml` which is a YAML file
containing metadata regarding the release. It has at least the following fields:
Each GitHub release contains an asset named `manifest.yaml` which is a YAML file
containing metadata regarding the release. The manifest is also included in the
root of an Enso version package. It has at least the following fields:
- `minimum-launcher-version` - specifies the minimum version of the launcher
that should be used with this release of Enso,
@ -106,7 +107,7 @@ graal-java-version: java11
The `minimum-launcher-version` should be updated whenever a new version of Enso
introduces changes that require a more recent launcher version. This value is
stored in
[`distribution/manifest.template.yml`](../../distribution/manifest.template.yml)
[`distribution/manifest.template.yaml`](../../distribution/manifest.template.yaml)
and other values are added to this template at build time.
#### Release Assets Structure
@ -122,7 +123,7 @@ platform. So each release should contain the following assets:
- `enso-launcher-<version>-linux-amd64.zip`
- `enso-launcher-<version>-macos-amd64.zip`
- `enso-launcher-<version>-windows-amd64.zip`
- `manifest.yml`
- `manifest.yaml`
#### Marking a Release as Broken
We intend to _never_ delete a release from GitHub, as users may have projects

View File

@ -1,10 +1,9 @@
{
"argv": [
"java",
"-XX:-UseJVMCIClassLoader",
"-Dgraalvm.locatorDisabled=true",
"-jar",
"<ABSOLUTE_PATH_TO_ENSO_JAR>",
"<ABSOLUTE_PATH_TO_RUNNER_JAR>",
"--jupyter-kernel",
"{connection_file}"
],