build: adding support for hercules ci effects

ci.nix

@@ -25,30 +25,24 @@ let
   serviceAccountKey = builtins.readFile
     ("/var/lib/hercules-ci-agent/secrets/service-account.json");
 
-  # Push a split output derivation containing "out" and "hash" outputs.
-  pushObject =
-    { name, extension, drv, contentType ? "application/octet-stream" }:
-    let
-      # Use the sha256 for the object key prefix.
-      sha256 = builtins.readFile (drv.hash + "/sha256");
-      # Use md5 as an idempotency check for gsutil.
-      contentMD5 = builtins.readFile (drv.hash + "/md5");
-    in localLib.pushStorageObject {
-      inherit serviceAccountKey name contentMD5 contentType;
+  # Push a derivation to a remote storage bucket as a post-build effect.
+  pushObject = { name, drv, contentType ? "application/octet-stream" }:
+    localLib.pushStorageObject {
+      inherit name contentType serviceAccountKey;
 
       bucket = "bootstrap.urbit.org";
-      object = "ci/${lib.removeSuffix extension name}${sha256}.${extension}";
+      object = "ci/${lib.removePrefix "/nix/store/" (toString drv)}";
       file = drv.out;
     };
 
-  # Build and push a split output pill derivation with the ".pill" file extension.
-  pushPill = name: pill:
-    pushObject {
-      inherit name;
+  # # Build and push a split output pill derivation with the ".pill" file extension.
+  # pushPill = name: pill:
+  #   pushObject {
+  #     inherit name;
 
-      drv = pill.build;
-      extension = "pill";
-    };
+  #     drv = pill.build;
+  #     extension = "pill";
+  #   };
 
   systems = lib.filterAttrs (_: v: builtins.elem v supportedSystems) {
     linux = "x86_64-linux";
@@ -99,11 +93,10 @@ in localLib.dimension "system" systems (systemName: system:
       in pushObject {
         name = tarball.name;
         drv = tarball;
-        extension = tarball.meta.extension;
         contentType = "application/x-gtar";
       };
     };
-      
+
     # Filter derivations that have meta.platform missing the current system,
     # such as testFakeShip on darwin.
     platformFilter = localLib.platformFilterGeneric system;

nix/lib/default.nix

@@ -12,7 +12,10 @@ let
 
     makeReleaseTarball = callPackage ./make-release-tarball.nix { };
 
-    pushStorageObject = callPackage ./push-storage-object.nix { };
+    pushStorageObject =
+      callPackage ./push-storage-object.nix { inherit makeEffect; };
+
+    makeEffect = callPackage ./effect { };
   };
 
 in fetchers // rec {

nix/lib/effect/default.nix

@@ -0,0 +1,122 @@
+{ cacert, curl, jq, lib, runCommandNoCC, stdenvNoCC }:
+
+let
+
+  defaultInputs = [ effectSetupHook cacert curl jq ];
+
+  mkDrv = args:
+    stdenvNoCC.mkDerivation (args // {
+
+      phases = args.phases or "initPhase unpackPhase patchPhase ${
+          args.preGetStatePhases or ""
+        } getStatePhase userSetupPhase ${
+          args.preEffectPhases or ""
+        } effectPhase putStatePhase ${args.postEffectPhases or ""}";
+
+      name = args.name + "-effect";
+
+      # nativeBuildInputs normally corresponds to what the building machine can
+      # execute. Likewise, effects are executed on the machine type that would
+      # otherwise perform the build.
+      # To keep things simple and avoid "build" terminology, we alias this as "inputs".
+      nativeBuildInputs = (args.defaultInputs or defaultInputs)
+        ++ (args.inputs or [ ]) ++ (args.nativeBuildInputs or [ ]);
+
+      isEffect = true;
+
+      # TODO: Use structured attrs instead
+      secretsMap = builtins.toJSON (args.secretsMap or { });
+
+    });
+
+  invokeOverride = f: defaults: (lib.makeOverridable f defaults).override;
+
+  effectSetupHook = runCommandNoCC "hercules-ci-effect-sh" { } ''
+    mkdir -p $out/nix-support
+    cp ${./effects-setup-hook.sh} $out/nix-support/setup-hook
+  '';
+
+in invokeOverride mkDrv {
+
+  preGetStatePhases = "";
+  preEffectPhases = "priorCheckPhase";
+
+  # Extension point for reporting notifications etc that are less criticial and
+  # don't write state.
+  postEffectPhases = "effectCheckPhase";
+
+  initPhase = ''
+    # eof on stdin
+    exec </dev/null
+    runHook preInit
+    eval "$initScript"
+    runHook postInit
+  '';
+  initScript = ''
+    export HOME=/build/home
+    mkdir -p $HOME
+    echo >/etc/passwd root:x:0:0:System administrator:/build/home:/run/current-system/sw/bin/bash
+    mkdir -p ~/.ssh
+    echo "BatchMode yes" >>~/.ssh/config
+  '';
+
+  userSetupPhase = ''
+    runHook preUserSetup
+    eval "$userSetupScript"
+    runHook postUserSetup
+  '';
+  userSetupScript = "";
+
+  # TODO Read a variable to optionally bail out if not already ok. That won't
+  #      permit a commit to fix a problem, which is why that isn't the current
+  #      behavior.
+  #      This variable can also be set dynamically by priorCheckScript to signal
+  #      that the problem is severe enough to abort the effect and thus require
+  #      manual intervention.
+  priorCheckPhase = ''
+    runHook prePriorCheck
+    if [[ -n "$priorCheckScript" ]]; then
+      if eval "$priorCheckScript"; then
+        echo 1>&2 -e 'Prior check \e[32;1mOK\e[0m.'
+      else
+        echo 1>&2
+        echo 1>&2 -e 'WARNING: Prior check \e[31;1mFAILED\e[0m!'
+        echo 1>&2
+        echo 1>&2 Continuing execution to allow subsequent steps to hopefully fix the problem.
+      fi
+    fi
+
+    runHook postPriorCheck
+  '';
+  priorCheckScript = "";
+
+  effectPhase = ''
+    runHook preEffect
+    eval "$effectScript"
+    runHook postEffect
+  '';
+  effectScript = "";
+
+  effectCheckPhase = ''
+    runHook preEffectCheck
+    eval "$effectCheckScript"
+    runHook postEffectCheck
+  '';
+
+  getStatePhase = ''
+    runHook preGetState
+    eval "$getStateScript"
+    runHook postGetState
+    registerPutStatePhaseOnFailure
+  '';
+  putStatePhase = ''
+    if [[ -z ''${PUT_STATE_DONE:-} ]]; then
+      runHook prePutState
+      eval "$putStateScript"
+      runHook postPutState
+      PUT_STATE_DONE=true
+    else
+      echo 1>&2 "NOTE: State has already been uploaded and was not uploaded again."
+    fi
+  '';
+}

nix/lib/effect/effects-setup-hook.sh

@@ -0,0 +1,173 @@
+# ----------------------------------------------------------------------------
+# prepare headers file for curl to talk to Hercules CI
+
+
+initHerculesCIAPI() {
+  herculesCIHeaders=$PWD/hercules-ci.headers
+  jq </secrets/secrets.json >$herculesCIHeaders -r '"Authorization: Bearer \(."hercules-ci".token)"'
+}
+preInitHooks+=("initHerculesCIAPI")
+
+
+# ----------------------------------------------------------------------------
+# state crud
+
+getStateFile() {
+  local stateName="$1"
+  local stateFileName="${2:-$1}"
+  echo 1>&2 "fetching state file $stateName"
+  while true; do
+    http_code=$(curl \
+      -H @$herculesCIHeaders \
+      --retry-max-time 86400 --retry-connrefused --max-time 1800 \
+      --silent --show-error \
+      --location \
+      "$HERCULES_CI_API_BASE_URL/api/v1/current-task/state/$stateName/data" \
+      -o "$stateFileName" \
+      -w '%{http_code}'
+      );
+    case $http_code in
+      200|204)
+        go_curl="false";
+        break ;;
+      408|421|429|5*)
+        echo 1>&2 "http status $http_code. Retrying..."
+        sleep 60
+        continue ;;
+      404)
+        echo 1>&2 "state file does not exist."
+        rm -f "$stateFileName"
+        break ;;
+      *)
+        echo 1>&2 "request failed with fatal status $http_code"
+        exit 1 ;;
+    esac
+  done
+}
+
+putStateFile() {
+  local stateName="$1"
+  local stateFileName="${2:-$1}"
+  echo "pushing state file $stateName..."
+  curl \
+    -H @$herculesCIHeaders \
+    --retry-max-time 86400 --retry-connrefused --max-time 1800 \
+    --silent --show-error \
+    --location --fail \
+    -XPUT \
+    --upload-file "$stateFileName" \
+    "$HERCULES_CI_API_BASE_URL/api/v1/current-task/state/$stateName/data" \
+    ;
+  echo "pushing state successful."
+}
+
+
+# ----------------------------------------------------------------------------
+# uploading state on error too
+
+putStatePhaseOnFailure() {
+  if [[ -n $putStatePhase ]]; then
+    echo 'uploading state files after failure' 1>&2
+    eval "$putStatePhase"
+  fi
+}
+
+registerPutStatePhaseOnFailure() {
+  failureHooks=("putStatePhaseOnFailure" "${failureHooks[@]}")
+}
+
+# ----------------------------------------------------------------------------
+# unpack fix
+
+
+simpleCopyUnpack() {
+  local fn="$1"
+  cp --no-preserve=ownership --recursive --reflink=auto \
+     -- $fn "$(stripHash "$fn")" \
+     ;
+}
+
+unpackCmdHooks+=(simpleCopyUnpack)
+
+
+# ----------------------------------------------------------------------------
+# warn if run in wrong environment
+
+
+if [[ "true" != ${IN_HERCULES_CI_EFFECT:-} ]]; then
+
+  if [[ -n ${NIX_LOG_FD:-} ]]; then
+    cat 1>&2 <<EOF
+WARNING: You are running a Hercules CI Effect in the Nix sandbox. This is very
+         unlikely to work. Effects are described in the derivation format and
+         have a lot in common, so you've probably tried to build it by accident.
+EOF
+  else
+    cat 1>&2 <<EOF
+WARNING: This effect is not running in the Hercules CI Effect sandbox.
+EOF
+  fi
+
+fi
+
+
+# ----------------------------------------------------------------------------
+# using secrets
+
+
+readSecretString() {
+  local secretName="$1"
+  local dataPath="$2"
+  if ! jq -e -r </secrets/secrets.json '.[$secretName] | '"$dataPath" --arg secretName "$secretName"
+  then echo 1>&2 "Could not find path $dataPath in secret $secretName"
+  fi
+}
+
+readSecretJSON() {
+  local secretName="$1"
+  local dataPath="$2"
+  jq -c </secrets/secrets.json '.[$secretName] | '"$dataPath" --arg secretName "$secretName"
+}
+
+writeAWSSecret() {
+  local secretName="${1:-aws}"
+  local profileName="${2:-default}"
+
+  mkdir -p ~/.aws
+  cat >>~/.aws/credentials <<EOF
+
+[$profileName]
+aws_secret_access_key = $(readSecretString "$secretName" .aws_secret_access_key)
+aws_access_key_id = $(readSecretString "$secretName" .aws_access_key_id)
+
+EOF
+}
+
+writeSSHKey() {
+  local secretName="${1:-ssh}"
+  local privateName="${2:-$HOME/.ssh/id_rsa}"
+  mkdir -p "$(dirname "$privateName")"
+  readSecretString "$secretName" .privateKey >"$privateName"
+  chmod 0400 "$privateName"
+  ssh-keygen -y -f "$privateName" >"$privateName.pub"
+}
+
+writeDockerKey() {
+  local secretName="${1:-docker}"
+  local directory="${2:-$HOME/.docker}"
+
+  mkdir -p $directory
+
+  readSecretString "$secretName" .clientKey >"$directory/key.pem"
+  readSecretString "$secretName" .clientCertificate >"$directory/cert.pem"
+  readSecretString "$secretName" .CACertificate >"$directory/ca.pem"
+
+  # Please permission checks if any
+  chmod 0400 "$directory"/{key,cert,ca}.pem
+}
+useDockerHost() {
+  local host="${1}"
+  local port="${2:-2376}"
+  export DOCKER_HOST=tcp://$host:$port
+  export DOCKER_TLS_VERIFY=1
+}

nix/lib/push-storage-object.nix

@@ -1,12 +1,6 @@
-{ lib, stdenvNoCC, coreutils, google-cloud-sdk, xxd }:
+{ lib, makeEffect, google-cloud-sdk }:
 
-# Somewhat annoyingly due to needing to use Google Storage's Content-MD5
-# to ensure a fixed output derivation - we need an md5sum of the file to
-# upload. This is in additional to any sha256sum you might want to actually
-# name the object key under.
-
-{ bucket, object, name, file, contentMD5, contentType, serviceAccountKey
-, preferLocalBuild ? true }:
+{ bucket, object, name, file, contentType, serviceAccountKey }:
 
 assert lib.asserts.assertMsg (builtins.isString serviceAccountKey)
   "`serviceAccountKey` must contain the JSON contents of a service-account key";
@@ -15,53 +9,33 @@ let
 
   uri = "gs://${bucket}/${lib.removePrefix "/" object}";
 
-in stdenvNoCC.mkDerivation {
-  name = "push-${lib.strings.sanitizeDerivationName name}";
+in makeEffect {
+  inherit file uri contentType serviceAccountKey;
 
-  nativeBuildInputs = [ coreutils google-cloud-sdk xxd ];
+  name = "push-${name}";
 
-  phases = [ "installPhase" ];
+  inputs = [ google-cloud-sdk ];
 
-  installPhase = ''
-    set -euo pipefail
+  dontUnpack = true;
 
+  effectScript = ''
     export HOME="."
 
-    gcloud auth activate-service-account --key-file=- <<< '${serviceAccountKey}'
+    gcloud auth activate-service-account --key-file=- <<< $serviceAccountKey
 
-    local_md5=$(echo -n '${contentMD5}' | xxd -r -p | base64)
-    remote_md5=
-       
     stat_uri() {
-      header "retrieving md5 for ${uri}"
-
-      remote_md5=$(gsutil stat '${uri}' \
-        | sed -n -e '/Hash (md5):/{s/.*: *//p}' \
-        | base64 -d \
-        | xxd -p)
+      gsutil stat $uri
     }
 
     if ! stat_uri; then
-       header "copying ${file} to ${uri}"
+       header "copying $file to $uri"
 
-       gsutil -h "Content-MD5:$local_md5" \
-              -h "Content-Type:${contentType}" \
-         cp '${file}' '${uri}'
+       gsutil -h "Content-Type:$contentType" cp $file $uri
 
        if ! stat_uri; then
-         echo "failed calculating remote uri md5" >&2
+         echo "failed pushing $file to $url" >&2
          exit 1 
        fi
     fi 
-
-    # This is the same format as md5sum (double space separator) and
-    # needs to match the .outputHash to ensure a fixed output derivation.
-    echo -n "$remote_md5  ${uri}" > $out
   '';
-
-  outputHashAlgo = "sha256";
-  outputHashMode = "flat";
-  outputHash = builtins.hashString "sha256" "${contentMD5}  ${uri}";
-
-  inherit preferLocalBuild;
 }