git.iko.soy/stackage
on master help sign in
stackage/automated/build.sh
158 lines 6.0 kB history raw
1#!/usr/bin/env bash
3set -eu +x -o pipefail
5ROOT=$(cd $(dirname $0) ; pwd)
6TARGET=$1
8source work/aws.sh
10# For nightly-YYYY-MM-DD, tag should be nightly
11# For lts-X.Y, tag should be ltsX
12SHORTNAME=$(echo $TARGET | cut -d- -f 1)
13if [ $SHORTNAME = "lts" ]
14then
15 TAG=$(echo $TARGET | sed 's@^lts-\([0-9]*\)\.[0-9]*@lts\1@')
16 WORKDIR=$ROOT/work/$(echo $TARGET | sed 's@^lts-\([0-9]*\)\.[0-9]*@lts-\1@')
17else
18 TAG=$SHORTNAME
19 WORKDIR=$ROOT/work/$TAG
20fi
22#IMAGE=commercialhaskell/stackage:$TAG
23IMAGE=ghcr.io/commercialhaskell/stackage/build:$TAG
25PANTRY_DIR=$ROOT/work/stack/pantry
26STACK_DIR=$ROOT/work/stack
27DOT_STACKAGE_DIR=$ROOT/work/dot-stackage
28# ssh key is used for committing snapshots (and their constraints) to Github
29SSH_DIR=$ROOT/work/ssh
30USERID=$(id -u)
32mkdir -p \
33 "$PANTRY_DIR" \
34 "$STACK_DIR" \
35 "$DOT_STACKAGE_DIR" \
36 "$WORKDIR" \
37 "$SSH_DIR"
39GITCONFIG=$ROOT/work/gitconfig
40cat >$GITCONFIG <<EOF
41[user]
42 email = michael+stackage-build@fpcomplete.com
43 name = Stackage Build host
44EOF
46HACKAGE_CREDS=$ROOT/work/hackage-creds
48function require_400_file {
49 if [ ! -f "$1" ]
50 then
51 echo File not found: "$1"
52 exit 1
53 fi
55 chmod 400 "$1"
58require_400_file "$SSH_DIR/id_rsa"
59require_400_file "$HACKAGE_CREDS"
61mkdir -p $ROOT/work/bin
62BINDIR=$(cd $ROOT/work/bin ; pwd)
64cd $BINDIR
65rm -f curator stack *.bz2
67curl -L "https://github.com/commercialhaskell/curator/releases/download/commit-1ef870178f6a0544fe5a3fd23a83fc075aabd695/curator.bz2" | bunzip2 > curator
68chmod +x curator
69echo -n "curator version: "
70docker run --rm -v $(pwd)/curator:/exe $IMAGE /exe --version
72curl -L https://github.com/commercialhaskell/stack/releases/download/v2.7.3/stack-2.7.3-linux-x86_64-bin > stack
73chmod +x stack
74echo -n "stack version: "
75docker run --rm -v $(pwd)/stack:/exe $IMAGE /exe --version
78# We share pantry directory between snapshots while the other content in .stack
79# is stored separately (because e.g. Ubuntu releases between LTS and nightly
80# could differ). Also the order of binds is important.
81ARGS_COMMON="--rm -v $WORKDIR:$HOME/work -w $HOME/work -v $BINDIR/curator:/usr/bin/curator:ro -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v $BINDIR/stack:/usr/bin/stack:ro -v $STACK_DIR:$HOME/.stack -v $PANTRY_DIR:$HOME/.stack/pantry"
82ARGS_PREBUILD="$ARGS_COMMON -u $USERID -e HOME=$HOME -v $DOT_STACKAGE_DIR:$HOME/.stackage"
83ARGS_BUILD="$ARGS_COMMON"
84# instance-data is an undocumented feature of S3 used by amazonka,
85# see https://github.com/brendanhay/amazonka/issues/271
86ARGS_UPLOAD="$ARGS_COMMON -u $USERID -e HOME=$HOME -v $HACKAGE_CREDS:/hackage-creds:ro -v $DOT_STACKAGE_DIR:$HOME/.stackage -v $SSH_DIR:$HOME/.ssh:ro -v $GITCONFIG:$HOME/.gitconfig:ro -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -v $DOT_STACKAGE_DIR:/dot-stackage"
88# Make sure we actually need this snapshot. We only check this for LTS releases
89# since, for nightlies, we'd like to run builds even if they are unnecessary to
90# get early warning information of upcoming failures. (See the duplicate check
91# below for why this is safe.)
92if [ $SHORTNAME = "lts" ]
93then
94 docker run $ARGS_UPLOAD $IMAGE /bin/bash -c "exec curator check-target-available --target $TARGET"
95fi
98# Determine the new build plan unless NOPLAN is set
100# * Update the package index (unless LTS)
101# * Create a new plan
102if [ "${NOPLAN:-}x" = "1x" ]
103then
104 docker run $ARGS_PREBUILD $IMAGE /bin/bash -c "curator snapshot-incomplete --target $TARGET && curator snapshot"
105elif [ "${NOPLAN:-}x" = "2x" ]
106then
107 docker run $ARGS_PREBUILD $IMAGE curator snapshot
108elif [ $SHORTNAME = "lts" ]
109then
110 docker run $ARGS_PREBUILD $IMAGE /bin/bash -c "curator constraints --target $TARGET && curator snapshot-incomplete --target $TARGET && curator snapshot"
111else
112 docker run $ARGS_PREBUILD $IMAGE /bin/bash -c "curator update && curator constraints --target $TARGET && curator snapshot-incomplete --target $TARGET && curator snapshot"
113fi
116# Do the rest of the pre-build actions:
118# * Check that the snapshot is valid
119# * Fetch and unpack all needed tarballs (the build step does not have write access to the tarball directory)
120docker run $ARGS_PREBUILD $IMAGE /bin/bash -c 'GHCVER=$(sed -n "s/^ghc-version: \(.*\)/\1/p" constraints.yaml) && stack setup ghc-$GHCVER --verbosity=error && stack exec --resolver=ghc-$GHCVER curator check-snapshot && curator unpack'
122case $SHORTNAME in
123 lts) JOBS=1 ;;
124 nightly) JOBS=2 ;;
125esac
127# Now do the actual build. We need to first set the owner of the home directory
128# correctly, so we run the command as root, change owner, and then use sudo to
129# switch back to the current user
130docker run $ARGS_BUILD $IMAGE nice -n 15 /bin/bash -c "chown $USER $HOME && exec sudo -E -u $USER env \"HOME=$HOME\" \"PATH=\$PATH\" curator build --jobs $JOBS" 2>&1 | tee "$SHORTNAME-build.log"
132# Make sure we actually need this snapshot. We used to perform this check
133# exclusively before building. Now we perform it after as well for the case of
134# nightly, where we don't perform this check beforehand. This is also slightly
135# safer, in case someone else already uploaded a specific snapshot while we
136# were building.
137docker run $ARGS_UPLOAD $IMAGE /bin/bash -c "exec curator check-target-available --target $TARGET"
139# Successful build, so we need to:
141# * Upload the docs to S3
142# * Upload the new snapshot .yaml file to the appropriate Github repo, also upload its constraints
143docker run $ARGS_UPLOAD $IMAGE /bin/bash -c "curator upload-docs --target $TARGET && curator upload-github --target $TARGET"
145# fixed in https://github.com/commercialhaskell/curator/pull/24
146docker run $ARGS_UPLOAD $IMAGE /bin/bash -c "exec curator hackage-distro --target $TARGET"
148# Build and push docker image fpco/stack-build & fpco/stack-build-small for current release
150if [ $SHORTNAME = "lts" ]
151then
152 $ROOT/dockerfiles/build.sh $TARGET
153 $ROOT/dockerfiles/build.sh --push $TARGET
154 $ROOT/dockerfiles/build.sh --push --small $TARGET
155fi
157echo -n "Completed at "
158date