From d3c5aa74c1ef9c4da6cf8c7474ec6079e95b973f Mon Sep 17 00:00:00 2001 From: Edward Amsden Date: Sat, 9 Jan 2021 11:04:45 -0500 Subject: [PATCH 1/2] build: A 'smart' docker image This commit adds the `docker-image` attribute to the main Nix entrypoint, invoking `nix/pkgs/docker-image` which will build a 'smart' docker image that can load keyfiles or a pier and boot a ship It includes a README for the official docker image, suitable for posting as the README to a Docker Hub or similar docker image repository. --- default.nix | 2 ++ nix/pkgs/docker-image/default.nix | 59 +++++++++++++++++++++++++++++++ pkg/docker-image/README.md | 39 ++++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 nix/pkgs/docker-image/default.nix create mode 100644 pkg/docker-image/README.md diff --git a/default.nix b/default.nix index b0029f1f70..57bfb8b469 100644 --- a/default.nix +++ b/default.nix @@ -115,6 +115,8 @@ let urbit = callPackage ./nix/pkgs/urbit { inherit enableStatic; }; + docker-image = callPackage ./nix/pkgs/docker-image { }; + hs = callPackage ./nix/pkgs/hs { inherit enableStatic; inherit (pkgsCross) haskell-nix; diff --git a/nix/pkgs/docker-image/default.nix b/nix/pkgs/docker-image/default.nix new file mode 100644 index 0000000000..8438fffc70 --- /dev/null +++ b/nix/pkgs/docker-image/default.nix @@ -0,0 +1,59 @@ +{ urbit, libcap, coreutils, bashInteractive, dockerTools, writeScriptBin, amesPort ? 34343 }: +let + startUrbit = writeScriptBin "start-urbit" '' + #!${bashInteractive}/bin/bash + + set -eu + + # Check if there is a keyfile, if so boot a ship with its name, and then remove the key + if [ -e *.key ]; then + # Get the name of the key + keynames="*.key" + keys=( $keynames ) + keyname=''${keys[0]} + mv $keyname /tmp + + # Boot urbit with the key, exit when done booting + urbit -w $(basename $keyname .key) -k /tmp/$keyname -c $(basename $keyname .key) -p ${toString amesPort} -x + + # Remove the keyfile for security + rm /tmp/$keyname + rm *.key || true + elif [ -e *.comet ]; then + cometnames="*.comet" + comets=( $cometnames ) + cometname=''${comets[0]} + rm *.comet + + urbit -c $(basename $cometname .comet) -p ${toString amesPort} -x + fi + + # Find the first directory and start urbit with the ship therein + dirnames="*/" + dirs=( $dirnames ) + dirname=''${dirnames[0]} + + urbit -p ${toString amesPort} $dirname + ''; + + +in dockerTools.buildImage { + name = "urbit"; + tag = "v${urbit.version}"; + contents = [ bashInteractive urbit startUrbit coreutils ]; + runAsRoot = '' + #!${bashInteractive} + mkdir -p /urbit + mkdir -p /tmp + ${libcap}/bin/setcap 'cap_net_bind_service=+ep' /bin/urbit + ''; + config = { + Cmd = [ "/bin/start-urbit" ]; + Env = [ "PATH=/bin" ]; + WorkingDir = "/urbit"; + Volumes = { + "/urbit" = {}; + }; + Expose = [ "80/tcp" "${toString amesPort}/udp" ]; + }; +} diff --git a/pkg/docker-image/README.md b/pkg/docker-image/README.md new file mode 100644 index 0000000000..5e22576226 --- /dev/null +++ b/pkg/docker-image/README.md @@ -0,0 +1,39 @@ +# Official Urbit Docker Image + +This is the official Docker image for [Urbit](https://urbit.org). + +Urbit is a clean-slate OS and network for the 21st century. + +## Using + +To use this image, you should mount a volume with a keyfile, comet file, or existing pier at `/urbit`, and map ports +as described below. + +### Volume Mount +This image expects a volume mounted at `/urbit`. This volume should initially obtain one of + +- A keyfile `.key` for a galaxy, star, planet, or moon. See the setup instructions for Urbit for information on [obtaining a keyfile](https://urbit.org/using/install/). + * e.g. `sampel-palnet.key` for the planet `sampel-palnet`. + to urbit to boot a ship from an existing pier, and in most cases (other than comets) will be named for the ship. +- An empty file with the extension `.comet`. This will cause Urbit to boot a [comet](https://urbit.org/docs/glossary/comet/) in a pier named for the `.comet` file (less the extension). + * e.g. starting with an empty file `my-urbit-bot.comet` will result in Urbit booting a comet into the pier + `my-urbit-bot` under your volume. +- An existing pier as a directory ``. You can migrate an existing ship to a new docker container in this way by placing its pier under the volume. + * e.g. if your ship is `sampel-palnet` then you likely have a directory `sampel-palnet` whose path you pass to `./urbit` when starting. While your ship is not running, move this directory to the volume and then start the container. + +The first two options result in Urbit attempting to boot either the ship named by the name of the keyfile, or a comet. In both cases, after that boot is successful, the `.key` or `.comet` file will be removed from the volume and the pier will take its place. + +In consequence, it is safe to remove the container and start a new container which mounts the same volume, e.g. to upgrade the version of the urbit binary by running a later container version. It is also possible to stop the container and then move the pier away e.g. to a location where you will run it directly with the Urbit binary. + +### Ports +The image includes `EXPOSE` directives for TCP port 80 and UDP port 34343. Port `80` is used for Urbit's HTTP interface for both [Landscape](https://urbit.org/docs/glossary/landscape/) and for [API calls](https://urbit.org/using/integrating-api/) to the ship. Port `34343` is used by [Ames](https://urbit.org/docs/glossary/ames/) for ship-to-ship communication. + +You can either pass the `-P` flag to docker to map ports directly to the corresponding ports on the host, or map them individually with `-p` flags. For local testing the latter is often convenient, for instance to remap port 80 to an unprivileged port. + +## Extending + +You likely do not want to extend this image. External applications which interact with Urbit do so primarily via an HTTP API, which should be exposed as described above. For containerized applications using Urbit, it is more appropriate to use a container orchestration service such as Docker Compose or Kubernetes to run Urbit alongside other containers which will interface with its API. + +## Development +The docker image is built by a Nix derivation in the [`nix/pkgs/docker-image/default.nix`](https://github.com/urbit/urbit/tree/master/nix/pkgs/docker-image/default.nix) file under the Urbit git repository. + From a64e0e141e2ac5e83f72560b24385e81d02781dc Mon Sep 17 00:00:00 2001 From: Edward Amsden Date: Wed, 13 Jan 2021 07:33:09 -0500 Subject: [PATCH 2/2] build: Build docker image and publish to Docker Hub d3c5aa74c1e introduced a Docker image intended for deploying Urbit ships This commit modifies the build.yml GitHub Actions workflow to build the docker image as part of the push/PR workflow, and adds a `release-docker.yml` workflow, which runs on any tag push `urbit-v*` and pushes the image to Docker Hub, also synchronizing the README. Co-authored-by: Josh Lehman --- .github/workflows/build.yml | 19 ++++++++++-- .github/workflows/release-docker.yml | 43 ++++++++++++++++++++++++++++ .github/workflows/release.yml | 2 +- default.nix | 2 ++ pkg/docker-image/README.md | 6 ++-- 5 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/release-docker.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7b89b59c58..dc7dacb200 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,10 +47,22 @@ jobs: steps: - uses: actions/checkout@v2 + + # We only want the extra nix config on linux, where it is necessary + # for the docker build. We don't want in on Mac, where it isn't but + # it breaks the nix install. The two `if` clauses should be mutually + # exclusive - uses: cachix/install-nix-action@v12 + with: + extra_nix_config: | + system-features = nixos-test benchmark big-parallel kvm + if: ${{ matrix.os == 'ubuntu-latest' }} + - uses: cachix/install-nix-action@v12 + if: ${{ matrix.os != 'ubuntu-latest' }} + - uses: cachix/cachix-action@v8 with: - name: ares + name: ${{ secrets.CACHIX_NAME }} authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - run: nix-build -A urbit --arg enableStatic true @@ -58,6 +70,9 @@ jobs: - if: ${{ matrix.os == 'ubuntu-latest' }} run: nix-build -A urbit-tests + - if: ${{ matrix.os == 'ubuntu-latest' }} + run: nix-build -A docker-image + haskell: strategy: fail-fast: false @@ -73,7 +88,7 @@ jobs: - uses: cachix/install-nix-action@v12 - uses: cachix/cachix-action@v8 with: - name: ares + name: ${{ secrets.CACHIX_NAME }} authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - run: nix-build -A hs.urbit-king.components.exes.urbit-king --arg enableStatic true diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml new file mode 100644 index 0000000000..36657b2231 --- /dev/null +++ b/.github/workflows/release-docker.yml @@ -0,0 +1,43 @@ +name: release-docker + +on: + release: null + push: + tags: ['urbit-v*'] + +jobs: + upload: + strategy: + matrix: + include: + - { os: ubuntu-latest, system: x86_64-linux } + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + - uses: cachix/install-nix-action@v12 + with: + extra_nix_config: | + system-features = nixos-test benchmark big-parallel kvm + - uses: cachix/cachix-action@v8 + with: + name: ${{ secrets.CACHIX_NAME }} + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + - uses: docker/docker-login-action@v1.8.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - uses: christian-korneck/update-container-description-action@v1 + env: + DOCKER_USER: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKER_PASS: ${{ secrets.DOCKERHUB_TOKEN }} + with: + destination_container_repo: ${{ secrets.DOCKERHUB_USERNAME }}/urbit + provider: dockerhub + short_description: 'Urbit: a clean-slate OS and network for the 21st century' + readme_file: 'pkg/docker-image/README.md' + + - run: | + version="$(cat ./pkg/urbit/version)" + $(nix-build -A skopeo)/bin/skopeo --insecure-policy copy tarball:$(nix-build -A docker-image) docker://${{ secrets.DOCKERHUB_USERNAME }}/urbit:v$version diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 78a8c01037..db22e4b8de 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ jobs: - uses: cachix/install-nix-action@v12 - uses: cachix/cachix-action@v8 with: - name: ares + name: ${{ secrets.CACHIX_NAME }} authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - uses: google-github-actions/setup-gcloud@v0.2.0 diff --git a/default.nix b/default.nix index 57bfb8b469..668b017025 100644 --- a/default.nix +++ b/default.nix @@ -160,6 +160,8 @@ let }; }; + inherit (pkgsNative) skopeo; + # A convenience function for constructing a shell.nix for any of the # pkgsLocal derivations by automatically propagating any dependencies # to the nix-shell. diff --git a/pkg/docker-image/README.md b/pkg/docker-image/README.md index 5e22576226..e5e2163eed 100644 --- a/pkg/docker-image/README.md +++ b/pkg/docker-image/README.md @@ -10,16 +10,15 @@ To use this image, you should mount a volume with a keyfile, comet file, or exis as described below. ### Volume Mount -This image expects a volume mounted at `/urbit`. This volume should initially obtain one of +This image expects a volume mounted at `/urbit`. This volume should initially contain one of - A keyfile `.key` for a galaxy, star, planet, or moon. See the setup instructions for Urbit for information on [obtaining a keyfile](https://urbit.org/using/install/). * e.g. `sampel-palnet.key` for the planet `sampel-palnet`. - to urbit to boot a ship from an existing pier, and in most cases (other than comets) will be named for the ship. - An empty file with the extension `.comet`. This will cause Urbit to boot a [comet](https://urbit.org/docs/glossary/comet/) in a pier named for the `.comet` file (less the extension). * e.g. starting with an empty file `my-urbit-bot.comet` will result in Urbit booting a comet into the pier `my-urbit-bot` under your volume. - An existing pier as a directory ``. You can migrate an existing ship to a new docker container in this way by placing its pier under the volume. - * e.g. if your ship is `sampel-palnet` then you likely have a directory `sampel-palnet` whose path you pass to `./urbit` when starting. While your ship is not running, move this directory to the volume and then start the container. + * e.g. if your ship is `sampel-palnet` then you likely have a directory `sampel-palnet` whose path you pass to `./urbit` when starting. [Move your pier](https://urbit.org/using/operations/using-your-ship/#moving-your-pier) directory to the volume and then start the container. The first two options result in Urbit attempting to boot either the ship named by the name of the keyfile, or a comet. In both cases, after that boot is successful, the `.key` or `.comet` file will be removed from the volume and the pier will take its place. @@ -36,4 +35,3 @@ You likely do not want to extend this image. External applications which interac ## Development The docker image is built by a Nix derivation in the [`nix/pkgs/docker-image/default.nix`](https://github.com/urbit/urbit/tree/master/nix/pkgs/docker-image/default.nix) file under the Urbit git repository. -