graphql-engine/.circleci/config.yml

# anchor refs to be used elsewhere
refs:
  skip_job_on_ciignore: &skip_job_on_ciignore
    run:
      name: checking if job should be terminated or not
      command: |
        if [ -f /build/ciignore/skip_job.txt ]; then
          echo "halting job due to /build/ciignore/skip_job.txt"
          circleci-agent step halt
        else
          echo "no skip_job file present, full steam ahead"
        fi        
  wait_for_postgres: &wait_for_postgres
    run:
      name: waiting for postgres to be ready
      command: |
        for i in `seq 1 60`;
        do
          nc -z localhost 5432 && echo Success && exit 0
          echo -n .
          sleep 1
        done
        echo Failed waiting for Postgres && exit 1        
  wait_for_hge: &wait_for_hge
    run:
      name: waiting for graphql-engine to be ready
      command: |
        for i in `seq 1 60`;
        do
          nc -z localhost 8080 && echo Success && exit 0
          echo -n .
          sleep 1
        done
        echo Failed waiting for graphql-engine && exit 1        

  filter_only_vtags: &filter_only_vtags
    filters:
      tags:
        only: /^v.*/
  filter_only_release_branches: &filter_only_release_branches
    filters:
      branches:
        only: /^release-v.*/
  filter_only_dev_branches: &filter_only_dev_branches
    filters:
      branches:
        only: /^dev.*/
  filter_only_vtags_dev_release_branches: &filter_only_vtags_dev_release_branches
    filters:
      tags:
        only: /^v.*/
      branches:
        only: /^(dev|release).*/
  filter_ignore_branches: &filter_ignore_branches
    filters:
      branches:
        ignore: /.*/
  filter_ignore_dev_release_branches: &filter_ignore_dev_release_branches
    filters:
      branches:
        ignore: /^(dev|release).*/

  setup_remote_docker: &setup_remote_docker
    setup_remote_docker:
      version: 17.09.0-ce
      docker_layer_caching: true

  # ref pg environment for testing
  test_pg_env: &test_pg_env
    environment:
      POSTGRES_USER: gql_test
      POSTGRES_DB: gql_test

  # ref test server job
  test_server: &test_server
    working_directory: ~/graphql-engine
    steps:
    - attach_workspace:
        at: /build
    - *skip_job_on_ciignore
    - checkout
    - *wait_for_postgres
    - run:
        name: test the server
        working_directory: ./server
        command: |
          DATABASE_URL="postgres://gql_test:@localhost:5432/gql_test" make ci-test          

  # ref pytest server job
  pytest_server: &pytest_server
    working_directory: ~/graphql-engine
    steps:
    - attach_workspace:
        at: /build
    - *skip_job_on_ciignore
    - checkout
    - restore_cache:
        keys:
        - server-app-cache-{{ .Branch }}-{{ .Revision }}
    - restore_cache:
        keys:
        - server-deps-cache-{{ checksum "server/graphql-engine.cabal" }}-{{ checksum "server/stack.yaml" }}
    - *wait_for_postgres
    - run:
        name: Install deps
        # if the man directories are missing, postgresql-client fails
        # to install in debian
        command: |
          mkdir -p /usr/share/man/man{1,7}
          apt-get update
          apt install --yes pgbouncer jq curl postgresql-client          
    - run:
        name: Ensure databases are present
        environment:
          DATABASE_URL: 'postgres://gql_test:@localhost:5432/gql_test'
        command: |
          psql "$DATABASE_URL" -c "SELECT 1 FROM pg_database WHERE datname = 'gql_test2'" | grep -q -F '(1 row)' || psql "$DATABASE_URL" -c 'CREATE DATABASE gql_test2;'          
    - run:
        name: Run Python tests
        environment:
          # hpc report seems to fail with the default -N
          GHCRTS: -N1
          HASURA_GRAPHQL_DATABASE_URL: 'postgres://gql_test:@localhost:5432/gql_test'
          HASURA_GRAPHQL_DATABASE_URL_2: 'postgres://gql_test:@localhost:5432/gql_test2'
          GRAPHQL_ENGINE: '/build/_server_output/graphql-engine'
        command: |
          OUTPUT_FOLDER=/build/_server_test_output/$PG_VERSION .circleci/test-server.sh          
    - run:
        name: Generate coverage report
        working_directory: ./server
        environment:
          LC_ALL: 'C.UTF-8'
        command: |
          stack --system-ghc hpc report /build/_server_test_output/$PG_VERSION/graphql-engine.tix --destdir /build/_server_test_output/$PG_VERSION          
    - store_artifacts:
        path: /build/_server_test_output
        destination: server_test

version: 2
jobs:
  # check if this should be built or not, fails if
  # changes only contains files in .ciignore
  check_build_worthiness:
    docker:
    - image: hasura/graphql-engine-cli-builder:v0.3
    working_directory: ~/graphql-engine
    steps:
    - attach_workspace:
        at: /build
    - checkout
    - run:
        name: check build worthiness
        command: .circleci/ciignore.sh
    - persist_to_workspace:
        root: /build
        paths:
        - ciignore

  # build the server binary, and package into docker image
  build_server:
    docker:
    - image: hasura/graphql-engine-server-builder:20190507-1
    working_directory: ~/graphql-engine
    steps:
    - attach_workspace:
        at: /build
    - *skip_job_on_ciignore
    - checkout
    - *setup_remote_docker
    - restore_cache:
        keys:
        - server-deps-cache-{{ checksum "server/graphql-engine.cabal" }}-{{ checksum "server/stack.yaml" }}
    - restore_cache:
        keys:
        - server-app-cache-{{ .Branch }}-{{ .Revision }}
    - run:
        name: Install latest postgresql client tools
        command: |
          apt-get -y update
          apt-get -y install postgresql-client-11          
    - run:
        name: Build the binary
        working_directory: ./server
        command: |
          # for PRs non-optimized build, else optimized build
          if [[ "$CIRCLE_BRANCH" =~ ^(dev|release) || "$CIRCLE_TAG" =~ ^v ]]; then
            echo "Branch starts with dev or release, or tagged commit starts with v. Optimized build"
            make ci-binary
          else
            echo "Non-release branch, non-optimized build with coverage"
            BUILD_FLAGS="--fast --coverage" make ci-binary
          fi          
    - run:
        name: Build the docker image
        working_directory: ./server
        command: |
          # copy console assets to the rootfs - packaging/build/rootfs
          export ROOTFS=packaging/build/rootfs
          mkdir -p "$ROOTFS/srv"
          cp -r /build/_console_output/assets "$ROOTFS/srv/console-assets"

          # build and save the image
          make ci-image
          make ci-save-image          
    - save_cache:
        key: server-app-cache-{{ .Branch }}-{{ .Revision }}
        paths:
        - ./server/.stack-work
    - save_cache:
        key: server-deps-cache-{{ checksum "server/graphql-engine.cabal" }}-{{ checksum "server/stack.yaml" }}
        paths:
        - ~/.stack
    - store_artifacts:
        path: /build/_server_output
        destination: server
    - persist_to_workspace:
        root: /build
        paths:
          - _server_output # binary is called graphql-engine


  # job to execute when all server tests pass. later we can collect test
  # reports and publish them etc.
  all_server_tests_pass:
    docker:
    - image: alpine:edge
    steps:
    - run:
        name: All server tests passed
        command: echo 'all server tests passed!'

  # pytest the server with postgres versions >= 9.5
  pytest_server_pg_11.1:
    <<: *pytest_server
    environment:
      PG_VERSION: "11_1"
    docker:
    - image: hasura/graphql-engine-server-builder:20190507-1
      # TODO: change this to circleci postgis when they have one for pg 11
    - image: mdillon/postgis:11-alpine
      <<: *test_pg_env

  pytest_server_pg_10.6:
    <<: *pytest_server
    environment:
      PG_VERSION: "10_6"
    docker:
    - image: hasura/graphql-engine-server-builder:20190507-1
    - image: circleci/postgres:10.6-alpine-postgis
      <<: *test_pg_env

  pytest_server_pg_9.6:
    <<: *pytest_server
    environment:
      PG_VERSION: "9_6"
    docker:
    - image: hasura/graphql-engine-server-builder:20190507-1
    - image: circleci/postgres:9.6-alpine-postgis
      <<: *test_pg_env

  pytest_server_pg_9.5:
    <<: *pytest_server
    environment:
      PG_VERSION: "9_5"
    docker:
    - image: hasura/graphql-engine-server-builder:20190507-1
    - image: circleci/postgres:9.5-alpine-postgis
      <<: *test_pg_env

  test_cli_with_last_release:
    docker:
    - image: hasura/graphql-engine-cli-builder:v0.3
    - image: circleci/postgres:10-alpine
      environment:
        POSTGRES_USER: gql_test
        POSTGRES_DB: gql_test
    working_directory: /go/src/github.com/hasura/graphql-engine
    steps:
    - attach_workspace:
        at: /build
    - *skip_job_on_ciignore
    - checkout
    - restore_cache:
        keys:
        - cli-vendor-{{ checksum "cli/Gopkg.toml" }}-{{ checksum "cli/Gopkg.lock" }}
    - run:
        name: get cli dependencies
        working_directory: cli
        command: make deps
    - save_cache:
        key: cli-vendor-{{ checksum "cli/Gopkg.toml" }}-{{ checksum "cli/Gopkg.lock" }}
        paths:
        - cli/vendor
    - *wait_for_postgres
    - run:
        name: test cli
        command: .circleci/test-cli-with-last-release.sh
    - store_artifacts:
        path: /build/_cli_output
        destination: cli

  # test and build cli
  test_and_build_cli:
    docker:
    - image: hasura/graphql-engine-cli-builder:v0.3
    - image: circleci/postgres:10-alpine
      environment:
        POSTGRES_USER: gql_test
        POSTGRES_DB: gql_test
    working_directory: /go/src/github.com/hasura/graphql-engine
    steps:
    - attach_workspace:
        at: /build
    - *skip_job_on_ciignore
    - checkout
    - restore_cache:
        keys:
        - cli-vendor-{{ checksum "cli/Gopkg.toml" }}-{{ checksum "cli/Gopkg.lock" }}
    - run:
        name: get cli dependencies
        working_directory: cli
        command: make deps
    - save_cache:
        key: cli-vendor-{{ checksum "cli/Gopkg.toml" }}-{{ checksum "cli/Gopkg.lock" }}
        paths:
        - cli/vendor
    - *wait_for_postgres
    - run:
        name: test cli
        command: .circleci/test-cli.sh
    - run:
        name: build cli
        working_directory: cli
        command: |
          make build
          make compress
          make ci-copy-binary          
    - store_artifacts:
        path: /build/_cli_output
        destination: cli
    - persist_to_workspace:
        root: /build
        paths:
          - _cli_output

  # build console assets
  build_console:
    docker:
    - image: hasura/graphql-engine-console-builder:20190515
    working_directory: ~/graphql-engine
    steps:
    - attach_workspace:
        at: /build
    - *skip_job_on_ciignore
    - checkout
    - restore_cache:
        key:
          console-npm-cache-{{ checksum "console/package.json" }}-{{ checksum "console/package-lock.json" }}
    - run:
        name: install dependencies
        working_directory: console
        command: make ci-deps
    - save_cache:
        key:
          console-npm-cache-{{ checksum "console/package.json" }}-{{ checksum "console/package-lock.json" }}
        paths:
        - console/node_modules
        - ~/.npm
        - ~/.cache
    - run:
        name: build console
        working_directory: console
        command: |
          make build
          make ci-copy-assets          
    - run:
        name: setup assets directory
        command: |
          export ASSETS_PATH=/build/_console_output/assets
          mkdir -p "$ASSETS_PATH"
          gsutil -m cp -r gs://graphql-engine-cdn.hasura.io/console/assets/common "$ASSETS_PATH"
          # gsutil decompresses files automatically, need to compress font-awesome again
          # (see https://github.com/GoogleCloudPlatform/gsutil/issues/515)
          mv "$ASSETS_PATH/common/css/font-awesome.min.css.gz" "$ASSETS_PATH/common/css/font-awesome.min.css"
          gzip "$ASSETS_PATH/common/css/font-awesome.min.css"
          # copy versioned assets and compress them
          mkdir -p "$ASSETS_PATH/versioned"
          cp "$ASSETS_PATH"/../{main.js,main.css,vendor.js} "$ASSETS_PATH/versioned/"
          gzip -r "$ASSETS_PATH/versioned/"          
    - store_artifacts:
        path: /build/_console_output
        destination: console
    - persist_to_workspace:
        root: /build
        paths:
          - _console_output

  # test console
  test_console:
    docker:
    - image: hasura/graphql-engine-console-builder:v0.3
      environment:
        CYPRESS_KEY: 983be0db-0f19-40cc-bfc4-194fcacd85e1
        GHCRTS: -N1
    - image: circleci/postgres:10-alpine-postgis
      environment:
        POSTGRES_USER: gql_test
        POSTGRES_DB: gql_test
    working_directory: ~/graphql-engine
    parallelism: 4
    steps:
    - attach_workspace:
        at: /build
    - *skip_job_on_ciignore
    - checkout
    - restore_cache:
        key:
          console-npm-cache-{{ checksum "console/package.json" }}-{{ checksum "console/package-lock.json" }}
    - run:
        name: install dependencies
        working_directory: console
        command: make ci-deps
    - save_cache:
        key:
          console-npm-cache-{{ checksum "console/package.json" }}-{{ checksum "console/package-lock.json" }}
        paths:
        - console/node_modules
        - ~/.npm
        - ~/.cache
    - *wait_for_postgres
    - run:
        name: test console
        command: .circleci/test-console.sh

  # test server upgrade from last version to current build
  test_server_upgrade:
    docker:
    - image: hasura/graphql-engine-upgrade-tester:v0.4
      environment:
        HASURA_GRAPHQL_DATABASE_URL: postgres://gql_test:@localhost:5432/gql_test
    - image: circleci/postgres:10-alpine
      environment:
        POSTGRES_USER: gql_test
        POSTGRES_DB: gql_test
    working_directory: ~/graphql-engine
    steps:
    - attach_workspace:
        at: /build
    - *skip_job_on_ciignore
    - checkout
    - run:
        name: upgrade_test
        command: .circleci/server-upgrade/run.sh
    - store_artifacts:
        path: /build/_server_output
        destination: server

  deploy:
    docker:
    - image: hasura/graphql-engine-deployer:v0.3
    working_directory: ~/graphql-engine
    steps:
    - attach_workspace:
        at: /build
    - *skip_job_on_ciignore
    - *setup_remote_docker
    - checkout
    - run:
        name: deploy
        command: .circleci/deploy.sh

workflows:
  version: 2
  workflow_v20190516:
    jobs:
    - check_build_worthiness: *filter_only_vtags
    - build_console:
        <<: *filter_only_vtags
        requires:
        - check_build_worthiness
    - build_server:
        <<: *filter_only_vtags
        requires:
        - build_console
    - pytest_server_pg_11.1:
        <<: *filter_only_vtags
        requires:
        - build_server
    - pytest_server_pg_10.6:
        <<: *filter_only_vtags
        requires:
        - build_server
    - pytest_server_pg_9.6:
        <<: *filter_only_vtags
        requires:
        - build_server
    - pytest_server_pg_9.5:
        <<: *filter_only_vtags
        requires:
        - build_server
    - test_server_upgrade:
        <<: *filter_only_vtags
        requires:
        - build_server
    - all_server_tests_pass:
        <<: *filter_only_vtags
        requires:
        - pytest_server_pg_11.1
        - pytest_server_pg_10.6
        - pytest_server_pg_9.6
        - pytest_server_pg_9.5
        - test_server_upgrade
    - test_cli_with_last_release:
        <<: *filter_only_vtags
        requires:
        - check_build_worthiness
    - test_and_build_cli:
        <<: *filter_only_vtags
        requires:
        - build_server
    - test_console:
        <<: *filter_only_vtags
        requires:
        - test_and_build_cli
        - test_cli_with_last_release
    - deploy:
        <<: *filter_only_vtags_dev_release_branches
        requires:
        - test_console
        - all_server_tests_pass