automated/build.sh · stackage

git.iko.soy/stackage

on 3218e4e help sign in

stackage/automated/build.sh

156 lines 5.9 kB history raw

1#!/usr/bin/env bash

3set -eu +x -o pipefail

5ROOT=$(cd $(dirname $0) ; pwd)

6TARGET=$1

8source work/aws.sh

10# For nightly-YYYY-MM-DD, tag should be nightly

11# For lts-X.Y, tag should be ltsX

12SHORTNAME=$(echo $TARGET | cut -d- -f 1)

13if [ $SHORTNAME = "lts" ]

14then

15 TAG=$(echo $TARGET | sed 's@^lts-$[0-9]*$\.[0-9]*@lts\1@')

16 WORKDIR=$ROOT/work/$(echo $TARGET | sed 's@^lts-$[0-9]*$\.[0-9]*@lts-\1@')

17else

18 TAG=$SHORTNAME

19 WORKDIR=$ROOT/work/$TAG

20fi

22IMAGE=commercialhaskell/stackage:$TAG

24PANTRY_DIR=$ROOT/work/stack/pantry

25STACK_DIR=$ROOT/work/stack

26DOT_STACKAGE_DIR=$ROOT/work/dot-stackage

27# ssh key is used for committing snapshots (and their constraints) to Github

28SSH_DIR=$ROOT/work/ssh

29USERID=$(id -u)

31mkdir -p \

32 "$PANTRY_DIR" \

33 "$STACK_DIR" \

34 "$DOT_STACKAGE_DIR" \

35 "$WORKDIR" \

36 "$SSH_DIR"

38GITCONFIG=$ROOT/work/gitconfig

39cat >$GITCONFIG <<EOF

40[user]

41 email = michael+stackage-build@fpcomplete.com

42 name = Stackage Build host

43EOF

45HACKAGE_CREDS=$ROOT/work/hackage-creds

47function require_400_file {

48 if [ ! -f "$1" ]

49 then

50 echo File not found: "$1"

51 exit 1

52 fi

54 chmod 400 "$1"

55}

57require_400_file "$SSH_DIR/id_rsa"

58require_400_file "$HACKAGE_CREDS"

60mkdir -p $ROOT/work/bin

61BINDIR=$(cd $ROOT/work/bin ; pwd)

62(

63cd $BINDIR

64rm -f curator stack *.bz2

66curl -L "https://download.fpcomplete.com/stackage-curator-2/curator-7c719d6d48839c94a79dc2ad2ace89074e3dd997.bz2" | bunzip2 > curator

67chmod +x curator

68echo -n "curator version: "

69docker run --rm -v $(pwd)/curator:/exe $IMAGE /exe --version

71curl -L "https://download.fpcomplete.com/stackage-curator-2/stack-fffc0a40e2253788f6b9cb7471c03fd571d69bde.bz2" | bunzip2 > stack

72chmod +x stack

73echo -n "stack version: "

74docker run --rm -v $(pwd)/stack:/exe $IMAGE /exe --version

75)

77# We share pantry directory between snapshots while the other content in .stack

78# is stored separately (because e.g. Ubuntu releases between LTS and nightly

79# could differ). Also the order of binds is important.

80ARGS_COMMON="--rm -v $WORKDIR:$HOME/work -w $HOME/work -v $BINDIR/curator:/usr/bin/curator:ro -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v $BINDIR/stack:/usr/bin/stack:ro -v $STACK_DIR:$HOME/.stack -v $PANTRY_DIR:$HOME/.stack/pantry"

81ARGS_PREBUILD="$ARGS_COMMON -u $USERID -e HOME=$HOME -v $DOT_STACKAGE_DIR:$HOME/.stackage"

82ARGS_BUILD="$ARGS_COMMON"

83# instance-data is an undocumented feature of S3 used by amazonka,

84# see https://github.com/brendanhay/amazonka/issues/271

85ARGS_UPLOAD="$ARGS_COMMON -u $USERID -e HOME=$HOME -v $HACKAGE_CREDS:/hackage-creds:ro -v $DOT_STACKAGE_DIR:$HOME/.stackage -v $SSH_DIR:$HOME/.ssh:ro -v $GITCONFIG:$HOME/.gitconfig:ro -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -v $DOT_STACKAGE_DIR:/dot-stackage"

87# Make sure we actually need this snapshot. We only check this for LTS releases

88# since, for nightlies, we'd like to run builds even if they are unnecessary to

89# get early warning information of upcoming failures. (See the duplicate check

90# below for why this is safe.)

91if [ $SHORTNAME = "lts" ]

92then

93 docker run $ARGS_UPLOAD $IMAGE /bin/bash -c "exec curator check-target-available --target $TARGET"

94fi

97# Determine the new build plan unless NOPLAN is set

98#

99# * Update the package index (unless LTS)

100# * Create a new plan

101if [ "${NOPLAN:-}x" = "x" ]

102then

103 if [ $SHORTNAME = "lts" ]

104 then

105 docker run $ARGS_PREBUILD $IMAGE /bin/bash -c "curator constraints --target $TARGET && curator snapshot-incomplete --target $TARGET && curator snapshot"

106 else

107 docker run $ARGS_PREBUILD $IMAGE /bin/bash -c "curator update && curator constraints --target $TARGET && curator snapshot-incomplete --target $TARGET && curator snapshot"

108 fi

109else

110 docker run $ARGS_PREBUILD $IMAGE /bin/bash -c "curator snapshot-incomplete --target $TARGET && curator snapshot"

111fi

112

113

114# Do the rest of the pre-build actions:

115#

116# * Check that the snapshot is valid

117# * Fetch and unpack all needed tarballs (the build step does not have write access to the tarball directory)

118docker run $ARGS_PREBUILD $IMAGE /bin/bash -c 'GHCVER=$(sed -n "s/^ghc-version: $.*$/\1/p" constraints.yaml) && stack setup ghc-$GHCVER --verbosity=error && stack exec --resolver=ghc-$GHCVER curator check-snapshot && curator unpack'

119

120case $SHORTNAME in

121 lts) JOBS=1 ;;

122 nightly) JOBS=2 ;;

123esac

124

125# Now do the actual build. We need to first set the owner of the home directory

126# correctly, so we run the command as root, change owner, and then use sudo to

127# switch back to the current user

128docker run $ARGS_BUILD $IMAGE nice -n 15 /bin/bash -c "chown $USER $HOME && exec sudo -E -u $USER env \"HOME=$HOME\" \"PATH=\$PATH\" curator build --jobs $JOBS" 2>&1 | tee "$SHORTNAME-build.log"

129

130# Make sure we actually need this snapshot. We used to perform this check

131# exclusively before building. Now we perform it after as well for the case of

132# nightly, where we don't perform this check beforehand. This is also slightly

133# safer, in case someone else already uploaded a specific snapshot while we

134# were building.

135docker run $ARGS_UPLOAD $IMAGE /bin/bash -c "exec curator check-target-available --target $TARGET"

136

137# Successful build, so we need to:

138#

139# * Upload the docs to S3

140# * Upload the new snapshot .yaml file to the appropriate Github repo, also upload its constraints

141docker run $ARGS_UPLOAD $IMAGE /bin/bash -c "curator upload-docs --target $TARGET && curator upload-github --target $TARGET"

142

143# For some reason, registering on Hackage fails with inscrutable error messages. Disabling.

144# docker run $ARGS_UPLOAD $IMAGE /bin/bash -c "exec curator hackage-distro --target $TARGET"

145

146# Build and push docker image fpco/stack-build & fpco/stack-build-small for current release

147

148if [ $SHORTNAME = "lts" ]

149then

150 $ROOT/dockerfiles/build.sh $TARGET

151 $ROOT/dockerfiles/build.sh --push $TARGET

152 $ROOT/dockerfiles/build.sh --push --small $TARGET

153fi

154

155echo -n "Completed at "

156date