953281e · stackage · git.iko.soy

git.iko.soy/stackage

More cleanups to build script

commit953281e263a51d77434d557b3f52490c9ffeff21 authorMichael Snoyman <michael@snoyman.com> dateSun 9 Feb 2020, 14:08 (7 years ago) parentb07c52c

11 files changed+10−233

compare with master

unifiedsplit

automated/.gitignore+5−8 automated/build-next.sh → automated/build.sh+4−3 automated/check-cabal.sh−72 automated/cron.sh−24 automated/new-stackage-format/convert.sh−35 automated/nix-build/README.md−5 automated/nix-build/build.sh−12 automated/nix-build/fetch-nixpkgs.nix−23 automated/nix-build/nixpkgs-src.json−6 automated/nix-build/snapshot-fixed.nix−41 automated/run-nightly.sh+1−4

automated/.gitignore+5−8

@@ -1,12 +1,9 @@

1−/auth-token

21/bin/

32/gitconfig

43/hackage-creds

5−/ssh-nightly/

6−/ssh-lts/

7−/nighlty/

8−/lts-*/

9−/stackage-curator/

10−/stackage-update/

4+/ssh/

5+/work/

116/aws.sh

12−/crondir/

7+/pantry/

8+/stack/

9+/dot-stackage/

automated/build-next.sh → automated/build.shrenamed+4−3

@@ -12,17 +12,18 @@ source aws.sh

1212SHORTNAME=$(echo $TARGET | cut -d- -f 1)

1313if [ $SHORTNAME = "lts" ]

1414then

15− TAG=$(echo $TARGET | sed 's@^lts-$[0-9]*$\.[0-9]*@lts\1@')

15+ TAG=$(echo $TARGET | sed 's@^lts-$[0-9]*$\.[0-9]*@lts-\1@')

16+ WORKDIR=$ROOT/work/$(echo $TARGET | sed 's@^lts-$[0-9]*$\.[0-9]*@lts\1@')

1617else

1718 TAG=$SHORTNAME

19+ WORKDIR=$ROOT/work/$TAG

1820fi

1921

2022IMAGE=commercialhaskell/stackage:$TAG

2123

22−PANTRY_DIR=$ROOT/pantry

24+PANTRY_DIR=$ROOT/stack/pantry

2325STACK_DIR=$ROOT/stack

2426DOT_STACKAGE_DIR=$ROOT/dot-stackage

25−WORKDIR=$ROOT/$TAG/work

2627# ssh key is used for committing snapshots (and their constraints) to Github

2728SSH_DIR=$ROOT/ssh

2829USERID=$(id -u)

automated/check-cabal.shdeleted−72

@@ -1,72 +0,0 @@

1−#!/usr/bin/env bash

2−

3−# This script is intended for testing Cabal HEAD against a Stackage snapshot.

4−

5−set -eux

6−

7−ROOT=$(cd $(dirname $0) ; pwd)

8−TARGET=$1

9−

10−# For nightly-YYYY-MM-DD, tag should be nightly

11−# For lts-X.Y, tag should be ltsX

12−SHORTNAME=$(echo $TARGET | cut -d- -f 1)

13−if [ $SHORTNAME = "lts" ]

14−then

15− TAG=$(echo $TARGET | sed 's@^lts-$[0-9]*$\.[0-9]*@lts\1@')

16− PLAN_URL=https://raw.githubusercontent.com/fpco/lts-haskell/master/$TARGET.yaml

17−else

18− TAG=$SHORTNAME

19− PLAN_URL=https://raw.githubusercontent.com/fpco/stackage-nightly/master/$TARGET.yaml

20−fi

21−

22−IMAGE=snoyberg/stackage:$TAG

23−

24−PLAN_FILE=$TARGET-plan.yaml

25−DOCMAP_FILE=$TARGET-docmap.yaml

26−BUNDLE_FILE=$TARGET.bundle=

27−

28−CABAL_DIR=$ROOT/cabal

29−STACK_DIR=$ROOT/stack

30−GHC_DIR=$ROOT/ghc

31−DOT_STACKAGE_DIR=$ROOT/dot-stackage

32−WORKDIR=$ROOT/$TAG/work

33−EXTRA_BIN_DIR=$ROOT/extra-bin

34−

35−mkdir -p \

36− "$CABAL_DIR" \

37− "$STACK_DIR" \

38− "$GHC_DIR" \

39− "$DOT_STACKAGE_DIR" \

40− "$WORKDIR" \

41− "$EXTRA_BIN_DIR"

42−

43−curl "$PLAN_URL" > $WORKDIR/$PLAN_FILE

44−

45−mkdir -p $ROOT/bin

46−BINDIR=$(cd $ROOT/bin ; pwd)

47−(

48−cd $BINDIR

49−rm -f stackage-curator stackage-curator.bz2

50−wget https://s3.amazonaws.com/stackage-travis/stackage-curator/stackage-curator.bz2

51−bunzip2 stackage-curator.bz2

52−chmod +x stackage-curator

53−)

54−

55−ARGS_COMMON="--rm -v $WORKDIR:/home/stackage/work -w /home/stackage/work -v $BINDIR/stackage-curator:/usr/bin/stackage-curator:ro -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v $EXTRA_BIN_DIR:/home/stackage/bin:ro"

56−ARGS_PREBUILD="$ARGS_COMMON -u $USER -v $CABAL_DIR:/home/stackage/.cabal -v $STACK_DIR:/home/stackage/.stack -v $GHC_DIR:/home/stackage/.ghc -v $DOT_STACKAGE_DIR:/home/stackage/.stackage"

57−ARGS_BUILD="$ARGS_COMMON -v $CABAL_DIR:/home/stackage/.cabal:ro -v $STACK_DIR:/home/stackage/.stack:ro -v $GHC_DIR:/home/stackage/.ghc:ro"

58−

59−# Get latest stack

60−curl -L https://www.stackage.org/stack/linux-x86_64 | tar xz --wildcards --strip-components=1 -C $EXTRA_BIN_DIR '*/stack'

61−

62−# Do all of the pre-build actions:

63−#

64−# * Update the package index

65−# * Fetch all needed tarballs (the build step does not have write access to the tarball directory)

66−# * Do a single unpack to create the package index cache (again due to directory perms)

67−docker run $ARGS_PREBUILD $IMAGE /bin/bash -c "/home/stackage/bin/stack update && stackage-curator fetch --plan-file $PLAN_FILE && cd /tmp && /home/stackage/bin/stack unpack random"

68−

69−# Now do the actual build. We need to first set the owner of the home directory

70−# correctly, so we run the command as root, change owner, and then use sudo to

71−# switch back to the current user

72−docker run $ARGS_BUILD $IMAGE /bin/bash -c "chown $USER /home/stackage && sudo -E -u $USER env \"PATH=\$PATH:/home/stackage/bin\" stackage-curator make-bundle --plan-file $PLAN_FILE --docmap-file $DOCMAP_FILE --bundle-file $BUNDLE_FILE --target $TARGET --cabal-from-head"

automated/cron.shdeleted−24

@@ -1,24 +0,0 @@

1−#!/usr/bin/env bash

2−

3−set -eux

4−

5−cd "$(dirname "${BASH_SOURCE[0]}")"

6−

7−CRONDIR=$(pwd)/crondir

8−mkdir -p $CRONDIR

9−source aws.sh

10−

11−IMAGE=fpco/stackage-server-prod:latest

12−docker pull $IMAGE

13−

14−stack update

15−

16−date

17−echo "Running stackage-server-cron..."

18−docker run --rm \

19− -v $CRONDIR:/home/ubuntu \

20− -v $HOME/.stack/indices:/home/ubuntu/.stack/indices:ro \

21− --workdir /home/ubuntu \

22− -p 17834:17834 \

23− $IMAGE \

24− bash -c "useradd $(whoami) -u $(id -u); sudo -u $(whoami) env HOME=/home/ubuntu AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY bash -c '/usr/local/bin/stackage-server-cron 2>&1 | tee -a /home/ubuntu/stackage-server-cron.log'"

automated/new-stackage-format/convert.shdeleted−35

@@ -1,35 +0,0 @@

1−#!/usr/bin/env bash

2−

3−set -eux

4−

5−if [[ ! -f convert ]]

6−then

7− curl https://s3.amazonaws.com/www.snoyman.com/convert-old-stackage-c9c4d06b31cb7aafedd23aa316b8008c45e4d4dd.bz2 > convert.bz2

8− chmod +x convert.bz2

9− bunzip2 convert.bz2

10−fi

11−

12−cd $(dirname ${BASH_SOURCE[0]})

13−

14−for d in lts-haskell stackage-nightly stackage-snapshots

15−do

16− if [[ ! -d "$d" ]]

17− then

18− git clone https://github.com/commercialhaskell/$d

19− else

20− (cd "$d" && git pull || echo "Git pull failed, ignoring")

21− fi

22−done

23−

24−(

25−export HOME=$(pwd)

26−./convert

27−)

28−

29−cd stackage-snapshots

30−git add lts nightly

31−git diff-index --quiet HEAD && echo No changes && exit 0

32−git config user.name "Stackage build server"

33−git config user.email "michael@snoyman.com"

34−git commit -m "More conversions $(date)"

35−GIT_SSH_COMMAND='ssh -i ../../ssh-lts/id_rsa' git push git@github.com:commercialhaskell/stackage-snapshots master

automated/nix-build/README.mddeleted−5

@@ -1,5 +0,0 @@

1−# Building Stackage with Nix

2−

3−The script `build.sh` is supposed to be run in a directory where `curator unpack`

4−was run - it assumes existence of `unpack-dir` directory with a Stackage

5−snapshot `stack.yaml` and unpacked packages in it.

automated/nix-build/build.shdeleted−12

@@ -1,12 +0,0 @@

1−#!/usr/bin/env bash

2−

3−set -eu +x

4−

5−NIX_VERSION="2.0.4"

6−curl https://nixos.org/releases/nix/nix-$NIX_VERSION/install | sh

7−

8−# no code to install stack2nix yet

9−

10−stack2nix --cabal2nix-args="--no-hpack" unpack-dir > snapshot.nix

11−

12−nix build -f snapshot-fixed.nix

automated/nix-build/fetch-nixpkgs.nixdeleted−23

@@ -1,23 +0,0 @@

1−let

2− spec = builtins.fromJSON (builtins.readFile ./nixpkgs-src.json);

3− src = import <nix/fetchurl.nix> {

4− url = "https://github.com/${spec.owner}/${spec.repo}/archive/${spec.rev}.tar.gz";

5− inherit (spec) sha256;

6− };

7− nixcfg = import <nix/config.nix>;

8−in builtins.derivation {

9− system = builtins.currentSystem;

10− name = "${src.name}-unpacked";

11− builder = builtins.storePath nixcfg.shell;

12− inherit src;

13− args = [

14− (builtins.toFile "builder" ''

15− $coreutils/mkdir $out

16− cd $out

17− $gzip -d < $src | $tar -x --strip-components=1

18− '')

19− ];

20− coreutils = builtins.storePath nixcfg.coreutils;

21− tar = builtins.storePath nixcfg.tar;

22− gzip = builtins.storePath nixcfg.gzip;

23−}

automated/nix-build/nixpkgs-src.jsondeleted−6

@@ -1,6 +0,0 @@

1−{

2− "owner": "NixOS",

3− "repo": "nixpkgs",

4− "rev": "c2950341d038995bf46a7b72db961bb3d3e9ac12",

5− "sha256": "0ghirjm6xvsihci3fladrn4nrnh2yr529v7c406hmw0yp25kv0i4"

6−}

automated/nix-build/snapshot-fixed.nixdeleted−41

@@ -1,41 +0,0 @@

1−{ pkgs ? import (import ./fetch-nixpkgs.nix) {}

2−, compiler ? pkgs.haskell.packages.ghc863

3−}:

4−let

5− snapshot = import ./snapshot.nix { pkgs = pkgs; compiler = compiler; };

6−in

7− snapshot.override (with pkgs.haskell.lib; {

8− overrides = self: super: {

9− # disabling packages from non-hackage-packages.nix

10− futhark = null;

11− multi-ghc-travis = null;

12− vaultenv = null;

13− # native deps

14− check-email = super.check-email.override { resolv = null; };

15− flac = super.flac.override { FLAC = pkgs.flac; };

16− freenect = super.freenect.override { freenect_sync = null; libfreenect = null; };

17− haskell-gi = super.haskell-gi.override { gobject-introspection = pkgs.gnome3.gobjectIntrospection; };

18− HDBC-mysql = super.HDBC-mysql.override { mysqlclient = pkgs.mysql;};

19− libffi = super.libffi.override { ffi = null; };

20− # fixes

21− blas-ffi = overrideCabal super.blas-ffi (

22− old: { libraryPkgconfigDepends = old.libraryPkgconfigDepends ++ [ pkgs.openblasCompat ]; }

23− );

24− odbc = overrideCabal super.odbc (

25− old: { librarySystemDepends = old.librarySystemDepends ++ [ pkgs.freetds ]; }

26− );

27− html-entities = null; # https://github.com/nikita-volkov/html-entities/issues/8

28− category = null; # conflict in Prelude.hs

29− constraint = null; # because of category

30− lame = null; # fails with 'Total ticks: 451489'

31− proto-lens-protobuf-types = overrideCabal super.proto-lens-protobuf-types (

32− old: { libraryToolDepends = old.libraryToolDepends ++ [ pkgs.protobuf ]; }

33− );

34− # libraft = null; # xxx/postgres/entries.sql: openFile: does not exist (No such file or directory) in libraft-0.2.0.0

35− xmlbf-xeno = null; # because of html-entities

36− xmonad = null; # xmonad requires extra patches for Nix

37− xmonad-contrib = null;

38− xmonad-extras = null;

39− hlibgit2 = disableHardening super.hlibgit2 [ "format" ];

40− };

41− })

automated/run-nightly.sh+1−4

@@ -3,12 +3,9 @@

33cd "$(dirname "${BASH_SOURCE[0]}")"

55while true; do

6− ./build-next.sh nightly-$(date -u +%F)

6+ ./build.sh nightly-$(date -u +%F)

77 date

9− #./new-stackage-format/convert.sh > /dev/null 2> /dev/null

10− #date

11−

129 sleep 30m

1310 echo

1411done