circleci integration (#74)

This commit is contained in:
Shahidh K Muhammed 2018-07-10 15:31:02 +05:30 committed by Anon Ray
parent b0e3cbdd6a
commit 50165fc1ed
12 changed files with 412 additions and 178 deletions

22
.circleci/build-docker-image.sh Executable file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -e
DF_VERSION=$1
for FILE in *.dockerfile; do
f=$(basename -- "$FILE")
f="${f%.*}"
TAG=hasura/graphql-engine-$f:$DF_VERSION
echo
echo "=============================================================="
echo "Building and pushing $TAG"
echo "=============================================================="
echo
echo "=======>>>> docker build -t $TAG -f $FILE ."
echo
docker build -t $TAG -f $FILE .
echo "=======>>>> docker push $TAG"
echo
docker push $TAG
done

View File

@ -1,17 +1,19 @@
FROM golang:1.10
# install go dependencies
RUN go get github.com/golang/dep/cmd/dep
RUN go get github.com/mitchellh/gox
RUN go get github.com/hasura/go-bindata/go-bindata
RUN go get github.com/tcnksm/ghr
ARG upx_version="3.94"
# install UPX
# install go dependencies
RUN go get github.com/golang/dep/cmd/dep \
&& go get github.com/mitchellh/gox \
&& go get github.com/hasura/go-bindata/go-bindata \
&& go get github.com/tcnksm/ghr
# install UPX and netcat
RUN apt-get update && apt-get install -y \
xz-utils \
xz-utils netcat libpq5 \
&& curl -Lo /tmp/upx-${upx_version}.tar.xz https://github.com/upx/upx/releases/download/v${upx_version}/upx-${upx_version}-amd64_linux.tar.xz \
&& xz -d -c /tmp/upx-${upx_version}.tar.xz \
| tar -xOf - upx-${upx_version}-amd64_linux/upx > /bin/upx \
&& chmod a+x /bin/upx \
&& apt-get -y auto-remove \
&& rm -rf /var/lib/apt/lists/*
ADD https://github.com/upx/upx/releases/download/v3.94/upx-3.94-amd64_linux.tar.xz /usr/local
RUN xz -d -c /usr/local/upx-3.94-amd64_linux.tar.xz | \
tar -xOf - upx-3.94-amd64_linux/upx > /bin/upx && \
chmod a+x /bin/upx
&& rm -rf /var/lib/apt/lists/*

View File

@ -1,186 +1,203 @@
# anchor refs to be used elsewhere
refs:
wait_for_postgres: &wait_for_postgres
run:
name: waiting for postgres to be ready
command: |
for i in `seq 1 60`;
do
nc -z localhost 5432 && echo Success && exit 0
echo -n .
sleep 1
done
echo Failed waiting for Postgres && exit 1
filter_only_vtags: &filter_only_vtags
filters:
tags:
only: /^v.*/
filter_only_release_branches: &filter_only_release_branches
filters:
branches:
only: /^release-v.*/
filter_only_dev_branches: &filter_only_dev_branches
filters:
branches:
only: /^dev.*/
filter_ignore_branches: &filter_ignore_branches
filters:
branches:
ignore: /.*/
filter_ignore_release_branches: &filter_ignore_release_branches
filters:
branches:
ignore: /^release-v.*/
version: 2
jobs:
# build (test) and push server to docker hub
server:
# test build the server binary
test_and_build_server:
docker:
- image: fpco/stack-build:lts-11.15
working_directory: /build/hasura/graphql-engine
steps:
- checkout
- setup_remote_docker:
version: 17.09.0-ce
docker_layer_caching: true
- run:
name: Setup - Install docker client and other build dependencies
command: |
set -ex
apt-get -y update && apt-get install -y make curl git upx
VER="17.09.0-ce"
curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz
tar -xz -C /tmp -f /tmp/docker-$VER.tgz
mv /tmp/docker/* /usr/bin
- restore_cache:
keys:
- server-app-cache-{{ .Branch }}-{{ .Revision }}
- server-deps-cache
- run:
name: Compile and build the binary
working_directory: ./server
command: |
make ci-binary
- save_cache:
key: server-app-cache-{{ .Branch }}-{{ .Revision }}
paths:
- server/.stack-work
- save_cache:
key: server-deps-cache
paths:
- ~/.stack
- run:
name: Package and build the docker image
working_directory: ./server
command: |
make ci-image
#- run:
# name: Test the image
# working_directory: ./server
# command: |
# make test
- run:
name: Login to docker hub
command: docker login -u "$DOCKER_USER" -p "$DOCKER_PASSWORD"
- run:
name: Push the image to registry
working_directory: ./server
command: |
make push
# build (test) and push console - version will be vMAJOR.MINOR for tags and release-v* branches
console:
docker:
- image: hasura/graphql-engine-console-builder:v0.1
- image: hasura/graphql-engine-server-builder:v0.2
- image: circleci/postgres:10-alpine
environment:
POSTGRES_USER: gql_test
POSTGRES_DB: gql_test
working_directory: ~/graphql-engine
steps:
- checkout
- restore_cache:
key:
console-node-modules-{{ checksum "console/package.json" }}
- run:
name: install dependencies
working_directory: ./console
command: make deps
- save_cache:
key:
console-node-modules-{{ checksum "console/package.json" }}
paths:
- console/node_modules
# run tests
- run:
name: build console
working_directory: ./console
command: make build
- run:
name: store gcloud service account to file
command: echo $GCLOUD_SERVICE_KEY > ${HOME}/gcloud-service-key.json
- run:
name: setup gcloud
command: |
gcloud auth activate-service-account --key-file=${HOME}/gcloud-service-key.json
gcloud --quiet config set project ${GOOGLE_PROJECT_ID}
- run:
name: copy assets to gcloud
working_directory: ./console
command: |
export VERSION=$(../scripts/get-version-circleci.sh)
make gcloud-cp-stable
make gcloud-set-metadata
# send pull requests to install manifests and heroku repo to update new server image tag
send_pr:
- checkout
- setup_remote_docker:
version: 17.09.0-ce
docker_layer_caching: true
- restore_cache:
keys:
- server-app-cache-{{ .Branch }}-{{ .Revision }}
- server-deps-cache-v1
- *wait_for_postgres
- run:
name: test and build server binary
working_directory: ./server
command: |
# TODO: make test
make ci-binary
make ci-image
make ci-save-image
- save_cache:
key: server-app-cache-{{ .Branch }}-{{ .Revision }}
paths:
- server/.stack-work
- save_cache:
key: server-deps-cache-v1
paths:
- ~/.stack
- store_artifacts:
path: /build/_server_output
destination: server
- persist_to_workspace:
root: /build
paths:
- _server_output # binary is called graphql-engine
# test and build cli
test_and_build_cli:
docker:
- image: busybox
steps:
- run: echo doing nothing for send_pr job
# upload cli binaries to github releases page for the tag
github_release:
docker:
- image: hasura/graphql-engine-cli-builder:v0.1
steps:
- attach_workspace:
at: /tmp/assets
- run:
name: upload assets
working_directory: /tmp/assets
command: ghr -draft -u "$GITHUB_USER" "$(ls .)" "$(ls .)"
# build (test) and upload cli binaries to circleci artifacts
cli:
docker:
- image: hasura/graphql-engine-cli-builder:v0.1
- image: hasura/graphql-engine-cli-builder:v0.2
- image: circleci/postgres:10-alpine
environment:
POSTGRES_USER: gql_test
POSTGRES_DB: gql_test
working_directory: /go/src/github.com/hasura/graphql-engine
steps:
- checkout
- attach_workspace:
at: /build
- restore_cache:
keys:
- cli-vendor-{{ checksum "cli/Gopkg.toml" }}-{{ checksum "cli/Gopkg.lock" }}
- run:
name: get cli dependencies
working_directory: /go/src/github.com/hasura/graphql-engine/cli
working_directory: cli
command: make deps
- save_cache:
key: cli-vendor-{{ checksum "cli/Gopkg.toml" }}-{{ checksum "cli/Gopkg.lock" }}
paths:
- cli/vendor
# - run:
# name: test cli
# command: .circleci/cli-test.sh
- *wait_for_postgres
- run:
name: test cli
command: .circleci/test-cli.sh
- run:
name: build cli
working_directory: /go/src/github.com/hasura/graphql-engine/cli
working_directory: cli
command: |
make build
make compress
make ci-copy-binary
- store_artifacts:
path: cli/_output
path: /build/_cli_output
destination: cli
- persist_to_workspace:
root: cli
root: /build
paths:
- _output
- _cli_output
# test and build console
test_and_build_console:
docker:
- image: hasura/graphql-engine-console-builder:v0.2
- image: circleci/postgres:10-alpine
environment:
POSTGRES_USER: gql_test
POSTGRES_DB: gql_test
working_directory: ~/graphql-engine
steps:
- checkout
- attach_workspace:
at: /build
- restore_cache:
key:
console-npm-cache-{{ checksum "console/package.json" }}-{{ checksum "console/package-lock.json" }}
- run:
name: install dependencies
working_directory: console
command: make ci-deps
- save_cache:
key:
console-npm-cache-{{ checksum "console/package.json" }}-{{ checksum "console/package-lock.json" }}
paths:
- console/node_modules
- ~/.npm
- ~/.cache
- *wait_for_postgres
# ignore console test for now
# - run:
# name: test console
# command: .circleci/test-console.sh
- run:
name: build console
working_directory: console
command: |
make build
make ci-copy-assets
- store_artifacts:
path: /build/_console_output
destination: console
- persist_to_workspace:
root: /build
paths:
- _console_output
deploy:
docker:
- image: hasura/graphql-engine-deployer:v0.2
working_directory: ~/graphql-engine
steps:
- setup_remote_docker:
version: 17.09.0-ce
docker_layer_caching: true
- checkout
- attach_workspace:
at: /build
- run:
name: deploy
command: .circleci/deploy.sh
workflows:
version: 2
# executed for all branches except release-v* and for all tags v*
# release jobs are only executed for v* tags
build_and_maybe_release:
build_and_test:
jobs:
- server: &filter_ignore_release_branches_only_vtags
filters:
branches:
ignore: /^release-v.*/
tags:
only: /^v.*/
- cli:
<< : *filter_ignore_release_branches_only_vtags
# requires:
# - server
- console:
<< : *filter_ignore_release_branches_only_vtags
# requires:
# - cli
- github_release:
filters: &only_vtags_ignore_branches
branches:
ignore: /.*/
tags:
only: /^v.*/
requires:
- cli
- send_pr:
filters: *only_vtags_ignore_branches
requires:
- server
# executed when release-v* branches are updated, to renew console assets
update_release:
jobs:
- console:
filters:
branches:
only: /^release-v.*/
- test_and_build_server: *filter_only_vtags
- test_and_build_cli:
<<: *filter_only_vtags
requires:
- test_and_build_server
- test_and_build_console:
<<: *filter_only_vtags
requires:
- test_and_build_server
- test_and_build_cli
- deploy:
<<: *filter_only_vtags
requires:
- test_and_build_console

View File

@ -2,9 +2,15 @@ FROM node:8
# update npm
RUN npm install -g npm@5
# install gcloud sdk
ADD https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-207.0.0-linux-x86_64.tar.gz /tmp
RUN tar -xzf /tmp/google-cloud-sdk-207.0.0-linux-x86_64.tar.gz -C /usr/local
RUN /usr/local/google-cloud-sdk/install.sh
ENV PATH "/usr/local/google-cloud-sdk/bin:$PATH"
# install dependencies
RUN apt-get update && apt-get install -y \
netcat \
libpq5 \
libgtk2.0-0 \
libnotify-dev \
libgconf-2-4 \
libnss3 \
libxss1 \
libasound2 \
xvfb \
&& rm -rf /var/lib/apt/lists/*

72
.circleci/deploy.sh Executable file
View File

@ -0,0 +1,72 @@
#!/usr/bin/env bash
set -eo pipefail
IFS=$'\n\t'
ROOT="$(readlink -f ${BASH_SOURCE[0]%/*}/../)"
## deploy functions
deploy_server() {
echo "deploying server"
cd "$ROOT/server"
docker login -u "$DOCKER_USER" -p "$DOCKER_PASSWORD"
make ci-load-image
make push
}
# TODO: open pull requests
draft_github_release() {
cd "$ROOT"
echo "drafting github release"
ghr -t "$GITHUB_TOKEN" \
-u "$CIRCLE_PROJECT_USERNAME" \
-r "$CIRCLE_PROJECT_REPONAME" \
-b "$CIRCLE_TAG" \
-draft \
"$CIRCLE_TAG" /build/_cli_output/
}
deploy_console() {
echo "deploying console"
echo $GCLOUD_SERVICE_KEY > ${HOME}/gcloud-service-key.json
gcloud auth activate-service-account --key-file=${HOME}/gcloud-service-key.json
gcloud --quiet config set project ${GOOGLE_PROJECT_ID}
cd "$ROOT/console"
export VERSION=$(../scripts/get-version-circleci.sh)
export DIST_PATH="/build/_console_output"
make gcloud-cp-stable
make gcloud-set-metadata
unset VERSION
unset DIST_PATH
}
# skip deploy for pull requests
if [[ -n "${CIRCLE_PR_NUMBER:-}" ]]; then
echo "not deploying for PRs"
exit
fi
# required env vars
# DOCKER_USER
# DOCKER_PASSWORD
# GITHUB_TOKEN
# GCLOUD_SERVICE_KEY
# CIRCLE_PROJECT_USERNAME
# CIRCLE_PROJECT_REPONAME
# CIRCLE_TAG
# CIRCLE_PR_NUMBER
# CIRCLE_BRANCH
RELEASE_BRANCH_REGEX="^release-v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$"
if [[ "$CIRCLE_BRANCH" =~ $RELEASE_BRANCH_REGEX ]]; then
# release branch, only update console
echo "release branch, only deploying console"
deploy_console
exit
fi
deploy_console
deploy_server
if [[ ! -z "$CIRCLE_TAG" ]]; then
draft_github_release
fi

View File

@ -0,0 +1,32 @@
FROM ubuntu:16.04
ARG docker_ver="17.09.0-ce"
ARG upx_version="3.94"
ARG gcloud_version="207.0.0"
ARG ghr_version="0.10.0"
RUN apt-get -y update \
&& apt-get -y upgrade \
&& apt-get install -y curl make xz-utils git python \
&& curl -Lo /tmp/docker-${docker_ver}.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${docker_ver}.tgz \
&& tar -xz -C /tmp -f /tmp/docker-${docker_ver}.tgz \
&& mv /tmp/docker/* /usr/bin \
&& curl -Lo /tmp/upx-${upx_version}.tar.xz https://github.com/upx/upx/releases/download/v${upx_version}/upx-${upx_version}-amd64_linux.tar.xz \
&& xz -d -c /tmp/upx-${upx_version}.tar.xz \
| tar -xOf - upx-${upx_version}-amd64_linux/upx > /bin/upx \
&& chmod a+x /bin/upx \
&& curl -Lo /tmp/ghr-${ghr_version}.tar.gz https://github.com/tcnksm/ghr/releases/download/v${ghr_version}/ghr_v${ghr_version}_linux_amd64.tar.gz \
&& tar -xz -C /tmp -f /tmp/ghr-${ghr_version}.tar.gz \
&& mv /tmp/ghr_v${ghr_version}_linux_amd64/* /usr/bin \
&& curl -Lo /tmp/gcloud-${gcloud_version}.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${gcloud_version}-linux-x86_64.tar.gz \
&& tar -xzf /tmp/gcloud-${gcloud_version}.tar.gz -C /usr/local \
&& /usr/local/google-cloud-sdk/install.sh \
&& apt-get -y purge curl \
&& apt-get -y auto-remove \
&& apt-get -y clean \
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf /usr/share/doc/ \
&& rm -rf /usr/share/man/ \
&& rm -rf /usr/share/locale/
ENV PATH "/usr/local/google-cloud-sdk/bin:$PATH"

View File

@ -6,12 +6,18 @@ ARG stack_ver="1.7.1"
# Install GNU make, curl, git and docker client. Required to build the server
RUN apt-get -y update \
&& apt-get install -y make curl git xz-utils \
&& apt-get -y upgrade \
&& apt-get install -y curl g++ gcc libc6-dev libpq-dev libffi-dev libgmp-dev make xz-utils zlib1g-dev git gnupg upx netcat \
&& curl -Lo /tmp/docker-${docker_ver}.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${docker_ver}.tgz \
&& tar -xz -C /tmp -f /tmp/docker-${docker_ver}.tgz \
&& mv /tmp/docker/* /usr/bin \
&& curl -sL https://github.com/commercialhaskell/stack/releases/download/v${stack_ver}/stack-${stack_ver}-linux-x86_64.tar.gz \
| tar xz --wildcards --strip-components=1 -C /usr/local/bin '*/stack' \
&& stack --resolver ${resolver} setup \
&& apt-get -y purge curl \
&& apt-get -y auto-remove \
&& apt-get -y clean \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf /usr/share/doc/ \
&& rm -rf /usr/share/man/ \
&& rm -rf /usr/share/locale/

18
.circleci/test-cli.sh Executable file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'
CLI_ROOT="${BASH_SOURCE[0]%/*}/../cli"
cd "$CLI_ROOT"
mkdir -p /build/_cli_output
touch /build/_cli_output/server.log
# start graphql-engine
/build/_server_output/graphql-engine \
--database-url postgres://gql_test@localhost:5432/gql_test serve > /build/_cli_output/server.log 2>&1 &
#PID=$!
# test cli
HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" make test
#kill $PID

34
.circleci/test-console.sh Executable file
View File

@ -0,0 +1,34 @@
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'
CONSOLE_ROOT="${BASH_SOURCE[0]%/*}/../console"
cd "$CONSOLE_ROOT"
mkdir -p /build/_console_output
touch /build/_console_output/server.log
touch /build/_console_output/cli.log
# start graphql-engine
/build/_server_output/graphql-engine \
--database-url postgres://gql_test@localhost:5432/gql_test serve > /build/_console_output/server.log 2>&1 &
# start cli
/build/_cli_output/hasura-linux-amd64 init --directory gql-test && cd gql-test
/build/_cli_output/hasura-linux-amd64 console --no-browser > /build/_console_output/cli.log 2>&1 &
cd ..
export PORT=3000
export NODE_ENV=development
export DATA_API_URL=http://localhost:8080
export API_HOST=http://localhost
export API_PORT=9693
export CONSOLE_MODE=cli
export DEV_DATA_API_URL=http://localhost:8080
export URL_PREFIX=/
# test console
npm run dev &
npm run test

View File

@ -40,10 +40,16 @@ build:
compress:
ls $(OUTPUT_DIR)/$(VERSION)/hasura-* | xargs upx
# to be executed in circle-ci only
ci-copy-binary:
mkdir -p /build/_cli_output
cp $(OUTPUT_DIR)/$(VERSION)/hasura-* /build/_cli_output
echo "$(VERSION)" > /build/_cli_output/version.txt
# run tests
.PHONY: test
test:
go test ./...
go test -ldflags "-X github.com/hasura/graphql-engine/cli/version.BuildVersion=$(VERSION)" ./...
# generate docs
.PHONY: docs

View File

@ -1,12 +1,15 @@
export PATH := node_modules/.bin:$(PATH)
DIST_PATH = ./static/dist
DIST_PATH ?= ./static/dist
BUCKET_NAME ?= hasura-graphql-engine
VERSION ?= $(shell ../scripts/get-version.sh)
all: deps build gcloud-cp-stable gcloud-set-metadata
deps:
if [ ! -d "node_modules" ]; then npm install; fi
npm install
ci-deps:
if [ ! -d "node_modules" ]; then npm ci; fi
build:
npm run build
@ -14,6 +17,12 @@ build:
test:
npm run test
# to be run inside circle-ci
ci-copy-assets:
mkdir -p /build/_console_output
cp $(DIST_PATH)/* /build/_console_output/
echo "$(VERSION)" > /build/_console_output/version.txt
gcloud-cp-stable:
gsutil cp -r $(DIST_PATH)/main.css gs://$(BUCKET_NAME)/console/$(VERSION)/main.css
gsutil cp -r $(DIST_PATH)/main.js gs://$(BUCKET_NAME)/console/$(VERSION)/main.js

View File

@ -7,6 +7,8 @@ packager_ver := 1.1
project_dir := $(shell pwd)
build_dir := $(project_dir)/$(shell stack --system-ghc path --dist-dir)/build
build_output := /build/_server_output
image: $(project).cabal
docker build -t "$(registry)/$(project):$(version)" \
-f packaging/Dockerfile \
@ -28,6 +30,9 @@ release-image: $(project).cabal
ci-binary: $(project).cabal
mkdir -p packaging/build/rootfs
stack --system-ghc build
mkdir -p $(build_output)
cp $(build_dir)/$(project)/$(project) $(build_output)
echo "$(version)" > $(build_output)/version.txt
# assumes this is built in circleci
ci-image:
@ -38,6 +43,11 @@ ci-image:
upx packaging/build/rootfs/bin/$(project)
docker build -t $(registry)/$(project):$(version) packaging/build/
ci-save-image:
docker save -o $(build_output)/image.tar $(registry)/$(project):$(version)
ci-load-image:
docker load -i $(build_output)/image.tar
push: $(project).cabal
docker push $(registry)/$(project):$(version)